From 18269c0fd4db9bec2668f895f21d742486ccb90f Mon Sep 17 00:00:00 2001 From: Sam Ravnborg Date: Wed, 3 Dec 2008 03:07:00 -0800 Subject: sparc: prepare lib/ for unification Identical named files renamed to _32.S Refactored Makefile to prepare for unification. Linking order was altered slightly - but this is a lib.a file so it should not matter. Signed-off-by: Sam Ravnborg Signed-off-by: David S. Miller --- arch/sparc/lib/Makefile | 25 +- arch/sparc/lib/atomic.S | 99 ------ arch/sparc/lib/atomic_32.S | 99 ++++++ arch/sparc/lib/checksum.S | 583 ---------------------------------- arch/sparc/lib/checksum_32.S | 583 ++++++++++++++++++++++++++++++++++ arch/sparc/lib/memcmp.S | 312 ------------------ arch/sparc/lib/memcmp_32.S | 312 ++++++++++++++++++ arch/sparc/lib/memscan.S | 133 -------- arch/sparc/lib/memscan_32.S | 133 ++++++++ arch/sparc/lib/rwsem.S | 204 ------------ arch/sparc/lib/rwsem_32.S | 204 ++++++++++++ arch/sparc/lib/strlen.S | 81 ----- arch/sparc/lib/strlen_32.S | 81 +++++ arch/sparc/lib/strlen_user.S | 109 ------- arch/sparc/lib/strlen_user_32.S | 109 +++++++ arch/sparc/lib/strncmp.S | 118 ------- arch/sparc/lib/strncmp_32.S | 118 +++++++ arch/sparc/lib/strncpy_from_user.S | 47 --- arch/sparc/lib/strncpy_from_user_32.S | 47 +++ 19 files changed, 1703 insertions(+), 1694 deletions(-) delete mode 100644 arch/sparc/lib/atomic.S create mode 100644 arch/sparc/lib/atomic_32.S delete mode 100644 arch/sparc/lib/checksum.S create mode 100644 arch/sparc/lib/checksum_32.S delete mode 100644 arch/sparc/lib/memcmp.S create mode 100644 arch/sparc/lib/memcmp_32.S delete mode 100644 arch/sparc/lib/memscan.S create mode 100644 arch/sparc/lib/memscan_32.S delete mode 100644 arch/sparc/lib/rwsem.S create mode 100644 arch/sparc/lib/rwsem_32.S delete mode 100644 arch/sparc/lib/strlen.S create mode 100644 arch/sparc/lib/strlen_32.S delete mode 100644 arch/sparc/lib/strlen_user.S create mode 100644 arch/sparc/lib/strlen_user_32.S delete mode 100644 arch/sparc/lib/strncmp.S create mode 100644 arch/sparc/lib/strncmp_32.S delete mode 100644 arch/sparc/lib/strncpy_from_user.S create mode 100644 arch/sparc/lib/strncpy_from_user_32.S (limited to 'arch/sparc/lib') diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile index 6e303e10c3b9..f2650545c774 100644 --- a/arch/sparc/lib/Makefile +++ b/arch/sparc/lib/Makefile @@ -1,13 +1,22 @@ # Makefile for Sparc library files.. # -EXTRA_AFLAGS := -ansi -DST_DIV0=0x02 +asflags-y := -ansi -DST_DIV0=0x02 +ccflags-y := -Werror -lib-y := mul.o rem.o sdiv.o udiv.o umul.o urem.o ashrdi3.o memcpy.o memset.o \ - strlen.o checksum.o blockops.o memscan.o memcmp.o strncmp.o \ - strncpy_from_user.o divdi3.o udivdi3.o strlen_user.o \ - copy_user.o locks.o atomic.o \ - lshrdi3.o ashldi3.o rwsem.o muldi3.o bitext.o \ - cmpdi2.o +lib-$(CONFIG_SPARC32) += mul.o rem.o sdiv.o udiv.o umul.o urem.o ashrdi3.o +lib-$(CONFIG_SPARC32) += memcpy.o memset.o +lib-y += strlen_$(BITS).o +lib-y += checksum_$(BITS).o +lib-$(CONFIG_SPARC32) += blockops.o +lib-y += memscan_$(BITS).o memcmp_$(BITS).o strncmp_$(BITS).o +lib-y += strncpy_from_user_$(BITS).o strlen_user_$(BITS).o +lib-$(CONFIG_SPARC32) += divdi3.o udivdi3.o +lib-$(CONFIG_SPARC32) += copy_user.o locks.o +lib-y += atomic_$(BITS).o +lib-$(CONFIG_SPARC32) += lshrdi3.o ashldi3.o +lib-y += rwsem_$(BITS).o +lib-$(CONFIG_SPARC32) += muldi3.o bitext.o cmpdi2.o -obj-y += iomap.o atomic32.o +obj-y += iomap.o +obj-$(CONFIG_SPARC32) += atomic32.o diff --git a/arch/sparc/lib/atomic.S b/arch/sparc/lib/atomic.S deleted file mode 100644 index 178cbb8ae1b9..000000000000 --- a/arch/sparc/lib/atomic.S +++ /dev/null @@ -1,99 +0,0 @@ -/* atomic.S: Move this stuff here for better ICACHE hit rates. - * - * Copyright (C) 1996 David S. Miller (davem@caipfs.rutgers.edu) - */ - -#include -#include - - .text - .align 4 - - .globl __atomic_begin -__atomic_begin: - -#ifndef CONFIG_SMP - .globl ___xchg32_sun4c -___xchg32_sun4c: - rd %psr, %g3 - andcc %g3, PSR_PIL, %g0 - bne 1f - nop - wr %g3, PSR_PIL, %psr - nop; nop; nop -1: - andcc %g3, PSR_PIL, %g0 - ld [%g1], %g7 - bne 1f - st %g2, [%g1] - wr %g3, 0x0, %psr - nop; nop; nop -1: - mov %g7, %g2 - jmpl %o7 + 8, %g0 - mov %g4, %o7 - - .globl ___xchg32_sun4md -___xchg32_sun4md: - swap [%g1], %g2 - jmpl %o7 + 8, %g0 - mov %g4, %o7 -#endif - - /* Read asm-sparc/atomic.h carefully to understand how this works for SMP. - * Really, some things here for SMP are overly clever, go read the header. - */ - .globl ___atomic24_add -___atomic24_add: - rd %psr, %g3 ! Keep the code small, old way was stupid - nop; nop; nop; ! Let the bits set - or %g3, PSR_PIL, %g7 ! Disable interrupts - wr %g7, 0x0, %psr ! Set %psr - nop; nop; nop; ! Let the bits set -#ifdef CONFIG_SMP -1: ldstub [%g1 + 3], %g7 ! Spin on the byte lock for SMP. - orcc %g7, 0x0, %g0 ! Did we get it? - bne 1b ! Nope... - ld [%g1], %g7 ! Load locked atomic24_t - sra %g7, 8, %g7 ! Get signed 24-bit integer - add %g7, %g2, %g2 ! Add in argument - sll %g2, 8, %g7 ! Transpose back to atomic24_t - st %g7, [%g1] ! Clever: This releases the lock as well. -#else - ld [%g1], %g7 ! Load locked atomic24_t - add %g7, %g2, %g2 ! Add in argument - st %g2, [%g1] ! Store it back -#endif - wr %g3, 0x0, %psr ! Restore original PSR_PIL - nop; nop; nop; ! Let the bits set - jmpl %o7, %g0 ! NOTE: not + 8, see callers in atomic.h - mov %g4, %o7 ! Restore %o7 - - .globl ___atomic24_sub -___atomic24_sub: - rd %psr, %g3 ! Keep the code small, old way was stupid - nop; nop; nop; ! Let the bits set - or %g3, PSR_PIL, %g7 ! Disable interrupts - wr %g7, 0x0, %psr ! Set %psr - nop; nop; nop; ! Let the bits set -#ifdef CONFIG_SMP -1: ldstub [%g1 + 3], %g7 ! Spin on the byte lock for SMP. - orcc %g7, 0x0, %g0 ! Did we get it? - bne 1b ! Nope... - ld [%g1], %g7 ! Load locked atomic24_t - sra %g7, 8, %g7 ! Get signed 24-bit integer - sub %g7, %g2, %g2 ! Subtract argument - sll %g2, 8, %g7 ! Transpose back to atomic24_t - st %g7, [%g1] ! Clever: This releases the lock as well -#else - ld [%g1], %g7 ! Load locked atomic24_t - sub %g7, %g2, %g2 ! Subtract argument - st %g2, [%g1] ! Store it back -#endif - wr %g3, 0x0, %psr ! Restore original PSR_PIL - nop; nop; nop; ! Let the bits set - jmpl %o7, %g0 ! NOTE: not + 8, see callers in atomic.h - mov %g4, %o7 ! Restore %o7 - - .globl __atomic_end -__atomic_end: diff --git a/arch/sparc/lib/atomic_32.S b/arch/sparc/lib/atomic_32.S new file mode 100644 index 000000000000..178cbb8ae1b9 --- /dev/null +++ b/arch/sparc/lib/atomic_32.S @@ -0,0 +1,99 @@ +/* atomic.S: Move this stuff here for better ICACHE hit rates. + * + * Copyright (C) 1996 David S. Miller (davem@caipfs.rutgers.edu) + */ + +#include +#include + + .text + .align 4 + + .globl __atomic_begin +__atomic_begin: + +#ifndef CONFIG_SMP + .globl ___xchg32_sun4c +___xchg32_sun4c: + rd %psr, %g3 + andcc %g3, PSR_PIL, %g0 + bne 1f + nop + wr %g3, PSR_PIL, %psr + nop; nop; nop +1: + andcc %g3, PSR_PIL, %g0 + ld [%g1], %g7 + bne 1f + st %g2, [%g1] + wr %g3, 0x0, %psr + nop; nop; nop +1: + mov %g7, %g2 + jmpl %o7 + 8, %g0 + mov %g4, %o7 + + .globl ___xchg32_sun4md +___xchg32_sun4md: + swap [%g1], %g2 + jmpl %o7 + 8, %g0 + mov %g4, %o7 +#endif + + /* Read asm-sparc/atomic.h carefully to understand how this works for SMP. + * Really, some things here for SMP are overly clever, go read the header. + */ + .globl ___atomic24_add +___atomic24_add: + rd %psr, %g3 ! Keep the code small, old way was stupid + nop; nop; nop; ! Let the bits set + or %g3, PSR_PIL, %g7 ! Disable interrupts + wr %g7, 0x0, %psr ! Set %psr + nop; nop; nop; ! Let the bits set +#ifdef CONFIG_SMP +1: ldstub [%g1 + 3], %g7 ! Spin on the byte lock for SMP. + orcc %g7, 0x0, %g0 ! Did we get it? + bne 1b ! Nope... + ld [%g1], %g7 ! Load locked atomic24_t + sra %g7, 8, %g7 ! Get signed 24-bit integer + add %g7, %g2, %g2 ! Add in argument + sll %g2, 8, %g7 ! Transpose back to atomic24_t + st %g7, [%g1] ! Clever: This releases the lock as well. +#else + ld [%g1], %g7 ! Load locked atomic24_t + add %g7, %g2, %g2 ! Add in argument + st %g2, [%g1] ! Store it back +#endif + wr %g3, 0x0, %psr ! Restore original PSR_PIL + nop; nop; nop; ! Let the bits set + jmpl %o7, %g0 ! NOTE: not + 8, see callers in atomic.h + mov %g4, %o7 ! Restore %o7 + + .globl ___atomic24_sub +___atomic24_sub: + rd %psr, %g3 ! Keep the code small, old way was stupid + nop; nop; nop; ! Let the bits set + or %g3, PSR_PIL, %g7 ! Disable interrupts + wr %g7, 0x0, %psr ! Set %psr + nop; nop; nop; ! Let the bits set +#ifdef CONFIG_SMP +1: ldstub [%g1 + 3], %g7 ! Spin on the byte lock for SMP. + orcc %g7, 0x0, %g0 ! Did we get it? + bne 1b ! Nope... + ld [%g1], %g7 ! Load locked atomic24_t + sra %g7, 8, %g7 ! Get signed 24-bit integer + sub %g7, %g2, %g2 ! Subtract argument + sll %g2, 8, %g7 ! Transpose back to atomic24_t + st %g7, [%g1] ! Clever: This releases the lock as well +#else + ld [%g1], %g7 ! Load locked atomic24_t + sub %g7, %g2, %g2 ! Subtract argument + st %g2, [%g1] ! Store it back +#endif + wr %g3, 0x0, %psr ! Restore original PSR_PIL + nop; nop; nop; ! Let the bits set + jmpl %o7, %g0 ! NOTE: not + 8, see callers in atomic.h + mov %g4, %o7 ! Restore %o7 + + .globl __atomic_end +__atomic_end: diff --git a/arch/sparc/lib/checksum.S b/arch/sparc/lib/checksum.S deleted file mode 100644 index 77f228533d47..000000000000 --- a/arch/sparc/lib/checksum.S +++ /dev/null @@ -1,583 +0,0 @@ -/* checksum.S: Sparc optimized checksum code. - * - * Copyright(C) 1995 Linus Torvalds - * Copyright(C) 1995 Miguel de Icaza - * Copyright(C) 1996 David S. Miller - * Copyright(C) 1997 Jakub Jelinek - * - * derived from: - * Linux/Alpha checksum c-code - * Linux/ix86 inline checksum assembly - * RFC1071 Computing the Internet Checksum (esp. Jacobsons m68k code) - * David Mosberger-Tang for optimized reference c-code - * BSD4.4 portable checksum routine - */ - -#include - -#define CSUM_BIGCHUNK(buf, offset, sum, t0, t1, t2, t3, t4, t5) \ - ldd [buf + offset + 0x00], t0; \ - ldd [buf + offset + 0x08], t2; \ - addxcc t0, sum, sum; \ - addxcc t1, sum, sum; \ - ldd [buf + offset + 0x10], t4; \ - addxcc t2, sum, sum; \ - addxcc t3, sum, sum; \ - ldd [buf + offset + 0x18], t0; \ - addxcc t4, sum, sum; \ - addxcc t5, sum, sum; \ - addxcc t0, sum, sum; \ - addxcc t1, sum, sum; - -#define CSUM_LASTCHUNK(buf, offset, sum, t0, t1, t2, t3) \ - ldd [buf - offset - 0x08], t0; \ - ldd [buf - offset - 0x00], t2; \ - addxcc t0, sum, sum; \ - addxcc t1, sum, sum; \ - addxcc t2, sum, sum; \ - addxcc t3, sum, sum; - - /* Do end cruft out of band to get better cache patterns. */ -csum_partial_end_cruft: - be 1f ! caller asks %o1 & 0x8 - andcc %o1, 4, %g0 ! nope, check for word remaining - ldd [%o0], %g2 ! load two - addcc %g2, %o2, %o2 ! add first word to sum - addxcc %g3, %o2, %o2 ! add second word as well - add %o0, 8, %o0 ! advance buf ptr - addx %g0, %o2, %o2 ! add in final carry - andcc %o1, 4, %g0 ! check again for word remaining -1: be 1f ! nope, skip this code - andcc %o1, 3, %o1 ! check for trailing bytes - ld [%o0], %g2 ! load it - addcc %g2, %o2, %o2 ! add to sum - add %o0, 4, %o0 ! advance buf ptr - addx %g0, %o2, %o2 ! add in final carry - andcc %o1, 3, %g0 ! check again for trailing bytes -1: be 1f ! no trailing bytes, return - addcc %o1, -1, %g0 ! only one byte remains? - bne 2f ! at least two bytes more - subcc %o1, 2, %o1 ! only two bytes more? - b 4f ! only one byte remains - or %g0, %g0, %o4 ! clear fake hword value -2: lduh [%o0], %o4 ! get hword - be 6f ! jmp if only hword remains - add %o0, 2, %o0 ! advance buf ptr either way - sll %o4, 16, %o4 ! create upper hword -4: ldub [%o0], %o5 ! get final byte - sll %o5, 8, %o5 ! put into place - or %o5, %o4, %o4 ! coalese with hword (if any) -6: addcc %o4, %o2, %o2 ! add to sum -1: retl ! get outta here - addx %g0, %o2, %o0 ! add final carry into retval - - /* Also do alignment out of band to get better cache patterns. */ -csum_partial_fix_alignment: - cmp %o1, 6 - bl cpte - 0x4 - andcc %o0, 0x2, %g0 - be 1f - andcc %o0, 0x4, %g0 - lduh [%o0 + 0x00], %g2 - sub %o1, 2, %o1 - add %o0, 2, %o0 - sll %g2, 16, %g2 - addcc %g2, %o2, %o2 - srl %o2, 16, %g3 - addx %g0, %g3, %g2 - sll %o2, 16, %o2 - sll %g2, 16, %g3 - srl %o2, 16, %o2 - andcc %o0, 0x4, %g0 - or %g3, %o2, %o2 -1: be cpa - andcc %o1, 0xffffff80, %o3 - ld [%o0 + 0x00], %g2 - sub %o1, 4, %o1 - addcc %g2, %o2, %o2 - add %o0, 4, %o0 - addx %g0, %o2, %o2 - b cpa - andcc %o1, 0xffffff80, %o3 - - /* The common case is to get called with a nicely aligned - * buffer of size 0x20. Follow the code path for that case. - */ - .globl csum_partial -csum_partial: /* %o0=buf, %o1=len, %o2=sum */ - andcc %o0, 0x7, %g0 ! alignment problems? - bne csum_partial_fix_alignment ! yep, handle it - sethi %hi(cpte - 8), %g7 ! prepare table jmp ptr - andcc %o1, 0xffffff80, %o3 ! num loop iterations -cpa: be 3f ! none to do - andcc %o1, 0x70, %g1 ! clears carry flag too -5: CSUM_BIGCHUNK(%o0, 0x00, %o2, %o4, %o5, %g2, %g3, %g4, %g5) - CSUM_BIGCHUNK(%o0, 0x20, %o2, %o4, %o5, %g2, %g3, %g4, %g5) - CSUM_BIGCHUNK(%o0, 0x40, %o2, %o4, %o5, %g2, %g3, %g4, %g5) - CSUM_BIGCHUNK(%o0, 0x60, %o2, %o4, %o5, %g2, %g3, %g4, %g5) - addx %g0, %o2, %o2 ! sink in final carry - subcc %o3, 128, %o3 ! detract from loop iters - bne 5b ! more to do - add %o0, 128, %o0 ! advance buf ptr - andcc %o1, 0x70, %g1 ! clears carry flag too -3: be cpte ! nope - andcc %o1, 0xf, %g0 ! anything left at all? - srl %g1, 1, %o4 ! compute offset - sub %g7, %g1, %g7 ! adjust jmp ptr - sub %g7, %o4, %g7 ! final jmp ptr adjust - jmp %g7 + %lo(cpte - 8) ! enter the table - add %o0, %g1, %o0 ! advance buf ptr -cptbl: CSUM_LASTCHUNK(%o0, 0x68, %o2, %g2, %g3, %g4, %g5) - CSUM_LASTCHUNK(%o0, 0x58, %o2, %g2, %g3, %g4, %g5) - CSUM_LASTCHUNK(%o0, 0x48, %o2, %g2, %g3, %g4, %g5) - CSUM_LASTCHUNK(%o0, 0x38, %o2, %g2, %g3, %g4, %g5) - CSUM_LASTCHUNK(%o0, 0x28, %o2, %g2, %g3, %g4, %g5) - CSUM_LASTCHUNK(%o0, 0x18, %o2, %g2, %g3, %g4, %g5) - CSUM_LASTCHUNK(%o0, 0x08, %o2, %g2, %g3, %g4, %g5) - addx %g0, %o2, %o2 ! fetch final carry - andcc %o1, 0xf, %g0 ! anything left at all? -cpte: bne csum_partial_end_cruft ! yep, handle it - andcc %o1, 8, %g0 ! check how much -cpout: retl ! get outta here - mov %o2, %o0 ! return computed csum - - .globl __csum_partial_copy_start, __csum_partial_copy_end -__csum_partial_copy_start: - -/* Work around cpp -rob */ -#define ALLOC #alloc -#define EXECINSTR #execinstr -#define EX(x,y,a,b) \ -98: x,y; \ - .section .fixup,ALLOC,EXECINSTR; \ - .align 4; \ -99: ba 30f; \ - a, b, %o3; \ - .section __ex_table,ALLOC; \ - .align 4; \ - .word 98b, 99b; \ - .text; \ - .align 4 - -#define EX2(x,y) \ -98: x,y; \ - .section __ex_table,ALLOC; \ - .align 4; \ - .word 98b, 30f; \ - .text; \ - .align 4 - -#define EX3(x,y) \ -98: x,y; \ - .section __ex_table,ALLOC; \ - .align 4; \ - .word 98b, 96f; \ - .text; \ - .align 4 - -#define EXT(start,end,handler) \ - .section __ex_table,ALLOC; \ - .align 4; \ - .word start, 0, end, handler; \ - .text; \ - .align 4 - - /* This aligned version executes typically in 8.5 superscalar cycles, this - * is the best I can do. I say 8.5 because the final add will pair with - * the next ldd in the main unrolled loop. Thus the pipe is always full. - * If you change these macros (including order of instructions), - * please check the fixup code below as well. - */ -#define CSUMCOPY_BIGCHUNK_ALIGNED(src, dst, sum, off, t0, t1, t2, t3, t4, t5, t6, t7) \ - ldd [src + off + 0x00], t0; \ - ldd [src + off + 0x08], t2; \ - addxcc t0, sum, sum; \ - ldd [src + off + 0x10], t4; \ - addxcc t1, sum, sum; \ - ldd [src + off + 0x18], t6; \ - addxcc t2, sum, sum; \ - std t0, [dst + off + 0x00]; \ - addxcc t3, sum, sum; \ - std t2, [dst + off + 0x08]; \ - addxcc t4, sum, sum; \ - std t4, [dst + off + 0x10]; \ - addxcc t5, sum, sum; \ - std t6, [dst + off + 0x18]; \ - addxcc t6, sum, sum; \ - addxcc t7, sum, sum; - - /* 12 superscalar cycles seems to be the limit for this case, - * because of this we thus do all the ldd's together to get - * Viking MXCC into streaming mode. Ho hum... - */ -#define CSUMCOPY_BIGCHUNK(src, dst, sum, off, t0, t1, t2, t3, t4, t5, t6, t7) \ - ldd [src + off + 0x00], t0; \ - ldd [src + off + 0x08], t2; \ - ldd [src + off + 0x10], t4; \ - ldd [src + off + 0x18], t6; \ - st t0, [dst + off + 0x00]; \ - addxcc t0, sum, sum; \ - st t1, [dst + off + 0x04]; \ - addxcc t1, sum, sum; \ - st t2, [dst + off + 0x08]; \ - addxcc t2, sum, sum; \ - st t3, [dst + off + 0x0c]; \ - addxcc t3, sum, sum; \ - st t4, [dst + off + 0x10]; \ - addxcc t4, sum, sum; \ - st t5, [dst + off + 0x14]; \ - addxcc t5, sum, sum; \ - st t6, [dst + off + 0x18]; \ - addxcc t6, sum, sum; \ - st t7, [dst + off + 0x1c]; \ - addxcc t7, sum, sum; - - /* Yuck, 6 superscalar cycles... */ -#define CSUMCOPY_LASTCHUNK(src, dst, sum, off, t0, t1, t2, t3) \ - ldd [src - off - 0x08], t0; \ - ldd [src - off - 0x00], t2; \ - addxcc t0, sum, sum; \ - st t0, [dst - off - 0x08]; \ - addxcc t1, sum, sum; \ - st t1, [dst - off - 0x04]; \ - addxcc t2, sum, sum; \ - st t2, [dst - off - 0x00]; \ - addxcc t3, sum, sum; \ - st t3, [dst - off + 0x04]; - - /* Handle the end cruft code out of band for better cache patterns. */ -cc_end_cruft: - be 1f - andcc %o3, 4, %g0 - EX(ldd [%o0 + 0x00], %g2, and %o3, 0xf) - add %o1, 8, %o1 - addcc %g2, %g7, %g7 - add %o0, 8, %o0 - addxcc %g3, %g7, %g7 - EX2(st %g2, [%o1 - 0x08]) - addx %g0, %g7, %g7 - andcc %o3, 4, %g0 - EX2(st %g3, [%o1 - 0x04]) -1: be 1f - andcc %o3, 3, %o3 - EX(ld [%o0 + 0x00], %g2, add %o3, 4) - add %o1, 4, %o1 - addcc %g2, %g7, %g7 - EX2(st %g2, [%o1 - 0x04]) - addx %g0, %g7, %g7 - andcc %o3, 3, %g0 - add %o0, 4, %o0 -1: be 1f - addcc %o3, -1, %g0 - bne 2f - subcc %o3, 2, %o3 - b 4f - or %g0, %g0, %o4 -2: EX(lduh [%o0 + 0x00], %o4, add %o3, 2) - add %o0, 2, %o0 - EX2(sth %o4, [%o1 + 0x00]) - be 6f - add %o1, 2, %o1 - sll %o4, 16, %o4 -4: EX(ldub [%o0 + 0x00], %o5, add %g0, 1) - EX2(stb %o5, [%o1 + 0x00]) - sll %o5, 8, %o5 - or %o5, %o4, %o4 -6: addcc %o4, %g7, %g7 -1: retl - addx %g0, %g7, %o0 - - /* Also, handle the alignment code out of band. */ -cc_dword_align: - cmp %g1, 6 - bl,a ccte - andcc %g1, 0xf, %o3 - andcc %o0, 0x1, %g0 - bne ccslow - andcc %o0, 0x2, %g0 - be 1f - andcc %o0, 0x4, %g0 - EX(lduh [%o0 + 0x00], %g4, add %g1, 0) - sub %g1, 2, %g1 - EX2(sth %g4, [%o1 + 0x00]) - add %o0, 2, %o0 - sll %g4, 16, %g4 - addcc %g4, %g7, %g7 - add %o1, 2, %o1 - srl %g7, 16, %g3 - addx %g0, %g3, %g4 - sll %g7, 16, %g7 - sll %g4, 16, %g3 - srl %g7, 16, %g7 - andcc %o0, 0x4, %g0 - or %g3, %g7, %g7 -1: be 3f - andcc %g1, 0xffffff80, %g0 - EX(ld [%o0 + 0x00], %g4, add %g1, 0) - sub %g1, 4, %g1 - EX2(st %g4, [%o1 + 0x00]) - add %o0, 4, %o0 - addcc %g4, %g7, %g7 - add %o1, 4, %o1 - addx %g0, %g7, %g7 - b 3f - andcc %g1, 0xffffff80, %g0 - - /* Sun, you just can't beat me, you just can't. Stop trying, - * give up. I'm serious, I am going to kick the living shit - * out of you, game over, lights out. - */ - .align 8 - .globl __csum_partial_copy_sparc_generic -__csum_partial_copy_sparc_generic: - /* %o0=src, %o1=dest, %g1=len, %g7=sum */ - xor %o0, %o1, %o4 ! get changing bits - andcc %o4, 3, %g0 ! check for mismatched alignment - bne ccslow ! better this than unaligned/fixups - andcc %o0, 7, %g0 ! need to align things? - bne cc_dword_align ! yes, we check for short lengths there - andcc %g1, 0xffffff80, %g0 ! can we use unrolled loop? -3: be 3f ! nope, less than one loop remains - andcc %o1, 4, %g0 ! dest aligned on 4 or 8 byte boundary? - be ccdbl + 4 ! 8 byte aligned, kick ass -5: CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x00,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) - CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x20,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) - CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x40,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) - CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x60,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) -10: EXT(5b, 10b, 20f) ! note for exception handling - sub %g1, 128, %g1 ! detract from length - addx %g0, %g7, %g7 ! add in last carry bit - andcc %g1, 0xffffff80, %g0 ! more to csum? - add %o0, 128, %o0 ! advance src ptr - bne 5b ! we did not go negative, continue looping - add %o1, 128, %o1 ! advance dest ptr -3: andcc %g1, 0x70, %o2 ! can use table? -ccmerge:be ccte ! nope, go and check for end cruft - andcc %g1, 0xf, %o3 ! get low bits of length (clears carry btw) - srl %o2, 1, %o4 ! begin negative offset computation - sethi %hi(12f), %o5 ! set up table ptr end - add %o0, %o2, %o0 ! advance src ptr - sub %o5, %o4, %o5 ! continue table calculation - sll %o2, 1, %g2 ! constant multiplies are fun... - sub %o5, %g2, %o5 ! some more adjustments - jmp %o5 + %lo(12f) ! jump into it, duff style, wheee... - add %o1, %o2, %o1 ! advance dest ptr (carry is clear btw) -cctbl: CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x68,%g2,%g3,%g4,%g5) - CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x58,%g2,%g3,%g4,%g5) - CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x48,%g2,%g3,%g4,%g5) - CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x38,%g2,%g3,%g4,%g5) - CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x28,%g2,%g3,%g4,%g5) - CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x18,%g2,%g3,%g4,%g5) - CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x08,%g2,%g3,%g4,%g5) -12: EXT(cctbl, 12b, 22f) ! note for exception table handling - addx %g0, %g7, %g7 - andcc %o3, 0xf, %g0 ! check for low bits set -ccte: bne cc_end_cruft ! something left, handle it out of band - andcc %o3, 8, %g0 ! begin checks for that code - retl ! return - mov %g7, %o0 ! give em the computed checksum -ccdbl: CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x00,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) - CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x20,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) - CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x40,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) - CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x60,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) -11: EXT(ccdbl, 11b, 21f) ! note for exception table handling - sub %g1, 128, %g1 ! detract from length - addx %g0, %g7, %g7 ! add in last carry bit - andcc %g1, 0xffffff80, %g0 ! more to csum? - add %o0, 128, %o0 ! advance src ptr - bne ccdbl ! we did not go negative, continue looping - add %o1, 128, %o1 ! advance dest ptr - b ccmerge ! finish it off, above - andcc %g1, 0x70, %o2 ! can use table? (clears carry btw) - -ccslow: cmp %g1, 0 - mov 0, %g5 - bleu 4f - andcc %o0, 1, %o5 - be,a 1f - srl %g1, 1, %g4 - sub %g1, 1, %g1 - EX(ldub [%o0], %g5, add %g1, 1) - add %o0, 1, %o0 - EX2(stb %g5, [%o1]) - srl %g1, 1, %g4 - add %o1, 1, %o1 -1: cmp %g4, 0 - be,a 3f - andcc %g1, 1, %g0 - andcc %o0, 2, %g0 - be,a 1f - srl %g4, 1, %g4 - EX(lduh [%o0], %o4, add %g1, 0) - sub %g1, 2, %g1 - srl %o4, 8, %g2 - sub %g4, 1, %g4 - EX2(stb %g2, [%o1]) - add %o4, %g5, %g5 - EX2(stb %o4, [%o1 + 1]) - add %o0, 2, %o0 - srl %g4, 1, %g4 - add %o1, 2, %o1 -1: cmp %g4, 0 - be,a 2f - andcc %g1, 2, %g0 - EX3(ld [%o0], %o4) -5: srl %o4, 24, %g2 - srl %o4, 16, %g3 - EX2(stb %g2, [%o1]) - srl %o4, 8, %g2 - EX2(stb %g3, [%o1 + 1]) - add %o0, 4, %o0 - EX2(stb %g2, [%o1 + 2]) - addcc %o4, %g5, %g5 - EX2(stb %o4, [%o1 + 3]) - addx %g5, %g0, %g5 ! I am now to lazy to optimize this (question it - add %o1, 4, %o1 ! is worthy). Maybe some day - with the sll/srl - subcc %g4, 1, %g4 ! tricks - bne,a 5b - EX3(ld [%o0], %o4) - sll %g5, 16, %g2 - srl %g5, 16, %g5 - srl %g2, 16, %g2 - andcc %g1, 2, %g0 - add %g2, %g5, %g5 -2: be,a 3f - andcc %g1, 1, %g0 - EX(lduh [%o0], %o4, and %g1, 3) - andcc %g1, 1, %g0 - srl %o4, 8, %g2 - add %o0, 2, %o0 - EX2(stb %g2, [%o1]) - add %g5, %o4, %g5 - EX2(stb %o4, [%o1 + 1]) - add %o1, 2, %o1 -3: be,a 1f - sll %g5, 16, %o4 - EX(ldub [%o0], %g2, add %g0, 1) - sll %g2, 8, %o4 - EX2(stb %g2, [%o1]) - add %g5, %o4, %g5 - sll %g5, 16, %o4 -1: addcc %o4, %g5, %g5 - srl %g5, 16, %o4 - addx %g0, %o4, %g5 - orcc %o5, %g0, %g0 - be 4f - srl %g5, 8, %o4 - and %g5, 0xff, %g2 - and %o4, 0xff, %o4 - sll %g2, 8, %g2 - or %g2, %o4, %g5 -4: addcc %g7, %g5, %g7 - retl - addx %g0, %g7, %o0 -__csum_partial_copy_end: - -/* We do these strange calculations for the csum_*_from_user case only, ie. - * we only bother with faults on loads... */ - -/* o2 = ((g2%20)&3)*8 - * o3 = g1 - (g2/20)*32 - o2 */ -20: - cmp %g2, 20 - blu,a 1f - and %g2, 3, %o2 - sub %g1, 32, %g1 - b 20b - sub %g2, 20, %g2 -1: - sll %o2, 3, %o2 - b 31f - sub %g1, %o2, %o3 - -/* o2 = (!(g2 & 15) ? 0 : (((g2 & 15) + 1) & ~1)*8) - * o3 = g1 - (g2/16)*32 - o2 */ -21: - andcc %g2, 15, %o3 - srl %g2, 4, %g2 - be,a 1f - clr %o2 - add %o3, 1, %o3 - and %o3, 14, %o3 - sll %o3, 3, %o2 -1: - sll %g2, 5, %g2 - sub %g1, %g2, %o3 - b 31f - sub %o3, %o2, %o3 - -/* o0 += (g2/10)*16 - 0x70 - * 01 += (g2/10)*16 - 0x70 - * o2 = (g2 % 10) ? 8 : 0 - * o3 += 0x70 - (g2/10)*16 - o2 */ -22: - cmp %g2, 10 - blu,a 1f - sub %o0, 0x70, %o0 - add %o0, 16, %o0 - add %o1, 16, %o1 - sub %o3, 16, %o3 - b 22b - sub %g2, 10, %g2 -1: - sub %o1, 0x70, %o1 - add %o3, 0x70, %o3 - clr %o2 - tst %g2 - bne,a 1f - mov 8, %o2 -1: - b 31f - sub %o3, %o2, %o3 -96: - and %g1, 3, %g1 - sll %g4, 2, %g4 - add %g1, %g4, %o3 -30: -/* %o1 is dst - * %o3 is # bytes to zero out - * %o4 is faulting address - * %o5 is %pc where fault occurred */ - clr %o2 -31: -/* %o0 is src - * %o1 is dst - * %o2 is # of bytes to copy from src to dst - * %o3 is # bytes to zero out - * %o4 is faulting address - * %o5 is %pc where fault occurred */ - save %sp, -104, %sp - mov %i5, %o0 - mov %i7, %o1 - mov %i4, %o2 - call lookup_fault - mov %g7, %i4 - cmp %o0, 2 - bne 1f - add %g0, -EFAULT, %i5 - tst %i2 - be 2f - mov %i0, %o1 - mov %i1, %o0 -5: - call __memcpy - mov %i2, %o2 - tst %o0 - bne,a 2f - add %i3, %i2, %i3 - add %i1, %i2, %i1 -2: - mov %i1, %o0 -6: - call __bzero - mov %i3, %o1 -1: - ld [%sp + 168], %o2 ! struct_ptr of parent - st %i5, [%o2] - ret - restore - - .section __ex_table,#alloc - .align 4 - .word 5b,2 - .word 6b,2 diff --git a/arch/sparc/lib/checksum_32.S b/arch/sparc/lib/checksum_32.S new file mode 100644 index 000000000000..77f228533d47 --- /dev/null +++ b/arch/sparc/lib/checksum_32.S @@ -0,0 +1,583 @@ +/* checksum.S: Sparc optimized checksum code. + * + * Copyright(C) 1995 Linus Torvalds + * Copyright(C) 1995 Miguel de Icaza + * Copyright(C) 1996 David S. Miller + * Copyright(C) 1997 Jakub Jelinek + * + * derived from: + * Linux/Alpha checksum c-code + * Linux/ix86 inline checksum assembly + * RFC1071 Computing the Internet Checksum (esp. Jacobsons m68k code) + * David Mosberger-Tang for optimized reference c-code + * BSD4.4 portable checksum routine + */ + +#include + +#define CSUM_BIGCHUNK(buf, offset, sum, t0, t1, t2, t3, t4, t5) \ + ldd [buf + offset + 0x00], t0; \ + ldd [buf + offset + 0x08], t2; \ + addxcc t0, sum, sum; \ + addxcc t1, sum, sum; \ + ldd [buf + offset + 0x10], t4; \ + addxcc t2, sum, sum; \ + addxcc t3, sum, sum; \ + ldd [buf + offset + 0x18], t0; \ + addxcc t4, sum, sum; \ + addxcc t5, sum, sum; \ + addxcc t0, sum, sum; \ + addxcc t1, sum, sum; + +#define CSUM_LASTCHUNK(buf, offset, sum, t0, t1, t2, t3) \ + ldd [buf - offset - 0x08], t0; \ + ldd [buf - offset - 0x00], t2; \ + addxcc t0, sum, sum; \ + addxcc t1, sum, sum; \ + addxcc t2, sum, sum; \ + addxcc t3, sum, sum; + + /* Do end cruft out of band to get better cache patterns. */ +csum_partial_end_cruft: + be 1f ! caller asks %o1 & 0x8 + andcc %o1, 4, %g0 ! nope, check for word remaining + ldd [%o0], %g2 ! load two + addcc %g2, %o2, %o2 ! add first word to sum + addxcc %g3, %o2, %o2 ! add second word as well + add %o0, 8, %o0 ! advance buf ptr + addx %g0, %o2, %o2 ! add in final carry + andcc %o1, 4, %g0 ! check again for word remaining +1: be 1f ! nope, skip this code + andcc %o1, 3, %o1 ! check for trailing bytes + ld [%o0], %g2 ! load it + addcc %g2, %o2, %o2 ! add to sum + add %o0, 4, %o0 ! advance buf ptr + addx %g0, %o2, %o2 ! add in final carry + andcc %o1, 3, %g0 ! check again for trailing bytes +1: be 1f ! no trailing bytes, return + addcc %o1, -1, %g0 ! only one byte remains? + bne 2f ! at least two bytes more + subcc %o1, 2, %o1 ! only two bytes more? + b 4f ! only one byte remains + or %g0, %g0, %o4 ! clear fake hword value +2: lduh [%o0], %o4 ! get hword + be 6f ! jmp if only hword remains + add %o0, 2, %o0 ! advance buf ptr either way + sll %o4, 16, %o4 ! create upper hword +4: ldub [%o0], %o5 ! get final byte + sll %o5, 8, %o5 ! put into place + or %o5, %o4, %o4 ! coalese with hword (if any) +6: addcc %o4, %o2, %o2 ! add to sum +1: retl ! get outta here + addx %g0, %o2, %o0 ! add final carry into retval + + /* Also do alignment out of band to get better cache patterns. */ +csum_partial_fix_alignment: + cmp %o1, 6 + bl cpte - 0x4 + andcc %o0, 0x2, %g0 + be 1f + andcc %o0, 0x4, %g0 + lduh [%o0 + 0x00], %g2 + sub %o1, 2, %o1 + add %o0, 2, %o0 + sll %g2, 16, %g2 + addcc %g2, %o2, %o2 + srl %o2, 16, %g3 + addx %g0, %g3, %g2 + sll %o2, 16, %o2 + sll %g2, 16, %g3 + srl %o2, 16, %o2 + andcc %o0, 0x4, %g0 + or %g3, %o2, %o2 +1: be cpa + andcc %o1, 0xffffff80, %o3 + ld [%o0 + 0x00], %g2 + sub %o1, 4, %o1 + addcc %g2, %o2, %o2 + add %o0, 4, %o0 + addx %g0, %o2, %o2 + b cpa + andcc %o1, 0xffffff80, %o3 + + /* The common case is to get called with a nicely aligned + * buffer of size 0x20. Follow the code path for that case. + */ + .globl csum_partial +csum_partial: /* %o0=buf, %o1=len, %o2=sum */ + andcc %o0, 0x7, %g0 ! alignment problems? + bne csum_partial_fix_alignment ! yep, handle it + sethi %hi(cpte - 8), %g7 ! prepare table jmp ptr + andcc %o1, 0xffffff80, %o3 ! num loop iterations +cpa: be 3f ! none to do + andcc %o1, 0x70, %g1 ! clears carry flag too +5: CSUM_BIGCHUNK(%o0, 0x00, %o2, %o4, %o5, %g2, %g3, %g4, %g5) + CSUM_BIGCHUNK(%o0, 0x20, %o2, %o4, %o5, %g2, %g3, %g4, %g5) + CSUM_BIGCHUNK(%o0, 0x40, %o2, %o4, %o5, %g2, %g3, %g4, %g5) + CSUM_BIGCHUNK(%o0, 0x60, %o2, %o4, %o5, %g2, %g3, %g4, %g5) + addx %g0, %o2, %o2 ! sink in final carry + subcc %o3, 128, %o3 ! detract from loop iters + bne 5b ! more to do + add %o0, 128, %o0 ! advance buf ptr + andcc %o1, 0x70, %g1 ! clears carry flag too +3: be cpte ! nope + andcc %o1, 0xf, %g0 ! anything left at all? + srl %g1, 1, %o4 ! compute offset + sub %g7, %g1, %g7 ! adjust jmp ptr + sub %g7, %o4, %g7 ! final jmp ptr adjust + jmp %g7 + %lo(cpte - 8) ! enter the table + add %o0, %g1, %o0 ! advance buf ptr +cptbl: CSUM_LASTCHUNK(%o0, 0x68, %o2, %g2, %g3, %g4, %g5) + CSUM_LASTCHUNK(%o0, 0x58, %o2, %g2, %g3, %g4, %g5) + CSUM_LASTCHUNK(%o0, 0x48, %o2, %g2, %g3, %g4, %g5) + CSUM_LASTCHUNK(%o0, 0x38, %o2, %g2, %g3, %g4, %g5) + CSUM_LASTCHUNK(%o0, 0x28, %o2, %g2, %g3, %g4, %g5) + CSUM_LASTCHUNK(%o0, 0x18, %o2, %g2, %g3, %g4, %g5) + CSUM_LASTCHUNK(%o0, 0x08, %o2, %g2, %g3, %g4, %g5) + addx %g0, %o2, %o2 ! fetch final carry + andcc %o1, 0xf, %g0 ! anything left at all? +cpte: bne csum_partial_end_cruft ! yep, handle it + andcc %o1, 8, %g0 ! check how much +cpout: retl ! get outta here + mov %o2, %o0 ! return computed csum + + .globl __csum_partial_copy_start, __csum_partial_copy_end +__csum_partial_copy_start: + +/* Work around cpp -rob */ +#define ALLOC #alloc +#define EXECINSTR #execinstr +#define EX(x,y,a,b) \ +98: x,y; \ + .section .fixup,ALLOC,EXECINSTR; \ + .align 4; \ +99: ba 30f; \ + a, b, %o3; \ + .section __ex_table,ALLOC; \ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4 + +#define EX2(x,y) \ +98: x,y; \ + .section __ex_table,ALLOC; \ + .align 4; \ + .word 98b, 30f; \ + .text; \ + .align 4 + +#define EX3(x,y) \ +98: x,y; \ + .section __ex_table,ALLOC; \ + .align 4; \ + .word 98b, 96f; \ + .text; \ + .align 4 + +#define EXT(start,end,handler) \ + .section __ex_table,ALLOC; \ + .align 4; \ + .word start, 0, end, handler; \ + .text; \ + .align 4 + + /* This aligned version executes typically in 8.5 superscalar cycles, this + * is the best I can do. I say 8.5 because the final add will pair with + * the next ldd in the main unrolled loop. Thus the pipe is always full. + * If you change these macros (including order of instructions), + * please check the fixup code below as well. + */ +#define CSUMCOPY_BIGCHUNK_ALIGNED(src, dst, sum, off, t0, t1, t2, t3, t4, t5, t6, t7) \ + ldd [src + off + 0x00], t0; \ + ldd [src + off + 0x08], t2; \ + addxcc t0, sum, sum; \ + ldd [src + off + 0x10], t4; \ + addxcc t1, sum, sum; \ + ldd [src + off + 0x18], t6; \ + addxcc t2, sum, sum; \ + std t0, [dst + off + 0x00]; \ + addxcc t3, sum, sum; \ + std t2, [dst + off + 0x08]; \ + addxcc t4, sum, sum; \ + std t4, [dst + off + 0x10]; \ + addxcc t5, sum, sum; \ + std t6, [dst + off + 0x18]; \ + addxcc t6, sum, sum; \ + addxcc t7, sum, sum; + + /* 12 superscalar cycles seems to be the limit for this case, + * because of this we thus do all the ldd's together to get + * Viking MXCC into streaming mode. Ho hum... + */ +#define CSUMCOPY_BIGCHUNK(src, dst, sum, off, t0, t1, t2, t3, t4, t5, t6, t7) \ + ldd [src + off + 0x00], t0; \ + ldd [src + off + 0x08], t2; \ + ldd [src + off + 0x10], t4; \ + ldd [src + off + 0x18], t6; \ + st t0, [dst + off + 0x00]; \ + addxcc t0, sum, sum; \ + st t1, [dst + off + 0x04]; \ + addxcc t1, sum, sum; \ + st t2, [dst + off + 0x08]; \ + addxcc t2, sum, sum; \ + st t3, [dst + off + 0x0c]; \ + addxcc t3, sum, sum; \ + st t4, [dst + off + 0x10]; \ + addxcc t4, sum, sum; \ + st t5, [dst + off + 0x14]; \ + addxcc t5, sum, sum; \ + st t6, [dst + off + 0x18]; \ + addxcc t6, sum, sum; \ + st t7, [dst + off + 0x1c]; \ + addxcc t7, sum, sum; + + /* Yuck, 6 superscalar cycles... */ +#define CSUMCOPY_LASTCHUNK(src, dst, sum, off, t0, t1, t2, t3) \ + ldd [src - off - 0x08], t0; \ + ldd [src - off - 0x00], t2; \ + addxcc t0, sum, sum; \ + st t0, [dst - off - 0x08]; \ + addxcc t1, sum, sum; \ + st t1, [dst - off - 0x04]; \ + addxcc t2, sum, sum; \ + st t2, [dst - off - 0x00]; \ + addxcc t3, sum, sum; \ + st t3, [dst - off + 0x04]; + + /* Handle the end cruft code out of band for better cache patterns. */ +cc_end_cruft: + be 1f + andcc %o3, 4, %g0 + EX(ldd [%o0 + 0x00], %g2, and %o3, 0xf) + add %o1, 8, %o1 + addcc %g2, %g7, %g7 + add %o0, 8, %o0 + addxcc %g3, %g7, %g7 + EX2(st %g2, [%o1 - 0x08]) + addx %g0, %g7, %g7 + andcc %o3, 4, %g0 + EX2(st %g3, [%o1 - 0x04]) +1: be 1f + andcc %o3, 3, %o3 + EX(ld [%o0 + 0x00], %g2, add %o3, 4) + add %o1, 4, %o1 + addcc %g2, %g7, %g7 + EX2(st %g2, [%o1 - 0x04]) + addx %g0, %g7, %g7 + andcc %o3, 3, %g0 + add %o0, 4, %o0 +1: be 1f + addcc %o3, -1, %g0 + bne 2f + subcc %o3, 2, %o3 + b 4f + or %g0, %g0, %o4 +2: EX(lduh [%o0 + 0x00], %o4, add %o3, 2) + add %o0, 2, %o0 + EX2(sth %o4, [%o1 + 0x00]) + be 6f + add %o1, 2, %o1 + sll %o4, 16, %o4 +4: EX(ldub [%o0 + 0x00], %o5, add %g0, 1) + EX2(stb %o5, [%o1 + 0x00]) + sll %o5, 8, %o5 + or %o5, %o4, %o4 +6: addcc %o4, %g7, %g7 +1: retl + addx %g0, %g7, %o0 + + /* Also, handle the alignment code out of band. */ +cc_dword_align: + cmp %g1, 6 + bl,a ccte + andcc %g1, 0xf, %o3 + andcc %o0, 0x1, %g0 + bne ccslow + andcc %o0, 0x2, %g0 + be 1f + andcc %o0, 0x4, %g0 + EX(lduh [%o0 + 0x00], %g4, add %g1, 0) + sub %g1, 2, %g1 + EX2(sth %g4, [%o1 + 0x00]) + add %o0, 2, %o0 + sll %g4, 16, %g4 + addcc %g4, %g7, %g7 + add %o1, 2, %o1 + srl %g7, 16, %g3 + addx %g0, %g3, %g4 + sll %g7, 16, %g7 + sll %g4, 16, %g3 + srl %g7, 16, %g7 + andcc %o0, 0x4, %g0 + or %g3, %g7, %g7 +1: be 3f + andcc %g1, 0xffffff80, %g0 + EX(ld [%o0 + 0x00], %g4, add %g1, 0) + sub %g1, 4, %g1 + EX2(st %g4, [%o1 + 0x00]) + add %o0, 4, %o0 + addcc %g4, %g7, %g7 + add %o1, 4, %o1 + addx %g0, %g7, %g7 + b 3f + andcc %g1, 0xffffff80, %g0 + + /* Sun, you just can't beat me, you just can't. Stop trying, + * give up. I'm serious, I am going to kick the living shit + * out of you, game over, lights out. + */ + .align 8 + .globl __csum_partial_copy_sparc_generic +__csum_partial_copy_sparc_generic: + /* %o0=src, %o1=dest, %g1=len, %g7=sum */ + xor %o0, %o1, %o4 ! get changing bits + andcc %o4, 3, %g0 ! check for mismatched alignment + bne ccslow ! better this than unaligned/fixups + andcc %o0, 7, %g0 ! need to align things? + bne cc_dword_align ! yes, we check for short lengths there + andcc %g1, 0xffffff80, %g0 ! can we use unrolled loop? +3: be 3f ! nope, less than one loop remains + andcc %o1, 4, %g0 ! dest aligned on 4 or 8 byte boundary? + be ccdbl + 4 ! 8 byte aligned, kick ass +5: CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x00,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) + CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x20,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) + CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x40,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) + CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x60,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) +10: EXT(5b, 10b, 20f) ! note for exception handling + sub %g1, 128, %g1 ! detract from length + addx %g0, %g7, %g7 ! add in last carry bit + andcc %g1, 0xffffff80, %g0 ! more to csum? + add %o0, 128, %o0 ! advance src ptr + bne 5b ! we did not go negative, continue looping + add %o1, 128, %o1 ! advance dest ptr +3: andcc %g1, 0x70, %o2 ! can use table? +ccmerge:be ccte ! nope, go and check for end cruft + andcc %g1, 0xf, %o3 ! get low bits of length (clears carry btw) + srl %o2, 1, %o4 ! begin negative offset computation + sethi %hi(12f), %o5 ! set up table ptr end + add %o0, %o2, %o0 ! advance src ptr + sub %o5, %o4, %o5 ! continue table calculation + sll %o2, 1, %g2 ! constant multiplies are fun... + sub %o5, %g2, %o5 ! some more adjustments + jmp %o5 + %lo(12f) ! jump into it, duff style, wheee... + add %o1, %o2, %o1 ! advance dest ptr (carry is clear btw) +cctbl: CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x68,%g2,%g3,%g4,%g5) + CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x58,%g2,%g3,%g4,%g5) + CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x48,%g2,%g3,%g4,%g5) + CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x38,%g2,%g3,%g4,%g5) + CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x28,%g2,%g3,%g4,%g5) + CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x18,%g2,%g3,%g4,%g5) + CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x08,%g2,%g3,%g4,%g5) +12: EXT(cctbl, 12b, 22f) ! note for exception table handling + addx %g0, %g7, %g7 + andcc %o3, 0xf, %g0 ! check for low bits set +ccte: bne cc_end_cruft ! something left, handle it out of band + andcc %o3, 8, %g0 ! begin checks for that code + retl ! return + mov %g7, %o0 ! give em the computed checksum +ccdbl: CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x00,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) + CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x20,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) + CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x40,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) + CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x60,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) +11: EXT(ccdbl, 11b, 21f) ! note for exception table handling + sub %g1, 128, %g1 ! detract from length + addx %g0, %g7, %g7 ! add in last carry bit + andcc %g1, 0xffffff80, %g0 ! more to csum? + add %o0, 128, %o0 ! advance src ptr + bne ccdbl ! we did not go negative, continue looping + add %o1, 128, %o1 ! advance dest ptr + b ccmerge ! finish it off, above + andcc %g1, 0x70, %o2 ! can use table? (clears carry btw) + +ccslow: cmp %g1, 0 + mov 0, %g5 + bleu 4f + andcc %o0, 1, %o5 + be,a 1f + srl %g1, 1, %g4 + sub %g1, 1, %g1 + EX(ldub [%o0], %g5, add %g1, 1) + add %o0, 1, %o0 + EX2(stb %g5, [%o1]) + srl %g1, 1, %g4 + add %o1, 1, %o1 +1: cmp %g4, 0 + be,a 3f + andcc %g1, 1, %g0 + andcc %o0, 2, %g0 + be,a 1f + srl %g4, 1, %g4 + EX(lduh [%o0], %o4, add %g1, 0) + sub %g1, 2, %g1 + srl %o4, 8, %g2 + sub %g4, 1, %g4 + EX2(stb %g2, [%o1]) + add %o4, %g5, %g5 + EX2(stb %o4, [%o1 + 1]) + add %o0, 2, %o0 + srl %g4, 1, %g4 + add %o1, 2, %o1 +1: cmp %g4, 0 + be,a 2f + andcc %g1, 2, %g0 + EX3(ld [%o0], %o4) +5: srl %o4, 24, %g2 + srl %o4, 16, %g3 + EX2(stb %g2, [%o1]) + srl %o4, 8, %g2 + EX2(stb %g3, [%o1 + 1]) + add %o0, 4, %o0 + EX2(stb %g2, [%o1 + 2]) + addcc %o4, %g5, %g5 + EX2(stb %o4, [%o1 + 3]) + addx %g5, %g0, %g5 ! I am now to lazy to optimize this (question it + add %o1, 4, %o1 ! is worthy). Maybe some day - with the sll/srl + subcc %g4, 1, %g4 ! tricks + bne,a 5b + EX3(ld [%o0], %o4) + sll %g5, 16, %g2 + srl %g5, 16, %g5 + srl %g2, 16, %g2 + andcc %g1, 2, %g0 + add %g2, %g5, %g5 +2: be,a 3f + andcc %g1, 1, %g0 + EX(lduh [%o0], %o4, and %g1, 3) + andcc %g1, 1, %g0 + srl %o4, 8, %g2 + add %o0, 2, %o0 + EX2(stb %g2, [%o1]) + add %g5, %o4, %g5 + EX2(stb %o4, [%o1 + 1]) + add %o1, 2, %o1 +3: be,a 1f + sll %g5, 16, %o4 + EX(ldub [%o0], %g2, add %g0, 1) + sll %g2, 8, %o4 + EX2(stb %g2, [%o1]) + add %g5, %o4, %g5 + sll %g5, 16, %o4 +1: addcc %o4, %g5, %g5 + srl %g5, 16, %o4 + addx %g0, %o4, %g5 + orcc %o5, %g0, %g0 + be 4f + srl %g5, 8, %o4 + and %g5, 0xff, %g2 + and %o4, 0xff, %o4 + sll %g2, 8, %g2 + or %g2, %o4, %g5 +4: addcc %g7, %g5, %g7 + retl + addx %g0, %g7, %o0 +__csum_partial_copy_end: + +/* We do these strange calculations for the csum_*_from_user case only, ie. + * we only bother with faults on loads... */ + +/* o2 = ((g2%20)&3)*8 + * o3 = g1 - (g2/20)*32 - o2 */ +20: + cmp %g2, 20 + blu,a 1f + and %g2, 3, %o2 + sub %g1, 32, %g1 + b 20b + sub %g2, 20, %g2 +1: + sll %o2, 3, %o2 + b 31f + sub %g1, %o2, %o3 + +/* o2 = (!(g2 & 15) ? 0 : (((g2 & 15) + 1) & ~1)*8) + * o3 = g1 - (g2/16)*32 - o2 */ +21: + andcc %g2, 15, %o3 + srl %g2, 4, %g2 + be,a 1f + clr %o2 + add %o3, 1, %o3 + and %o3, 14, %o3 + sll %o3, 3, %o2 +1: + sll %g2, 5, %g2 + sub %g1, %g2, %o3 + b 31f + sub %o3, %o2, %o3 + +/* o0 += (g2/10)*16 - 0x70 + * 01 += (g2/10)*16 - 0x70 + * o2 = (g2 % 10) ? 8 : 0 + * o3 += 0x70 - (g2/10)*16 - o2 */ +22: + cmp %g2, 10 + blu,a 1f + sub %o0, 0x70, %o0 + add %o0, 16, %o0 + add %o1, 16, %o1 + sub %o3, 16, %o3 + b 22b + sub %g2, 10, %g2 +1: + sub %o1, 0x70, %o1 + add %o3, 0x70, %o3 + clr %o2 + tst %g2 + bne,a 1f + mov 8, %o2 +1: + b 31f + sub %o3, %o2, %o3 +96: + and %g1, 3, %g1 + sll %g4, 2, %g4 + add %g1, %g4, %o3 +30: +/* %o1 is dst + * %o3 is # bytes to zero out + * %o4 is faulting address + * %o5 is %pc where fault occurred */ + clr %o2 +31: +/* %o0 is src + * %o1 is dst + * %o2 is # of bytes to copy from src to dst + * %o3 is # bytes to zero out + * %o4 is faulting address + * %o5 is %pc where fault occurred */ + save %sp, -104, %sp + mov %i5, %o0 + mov %i7, %o1 + mov %i4, %o2 + call lookup_fault + mov %g7, %i4 + cmp %o0, 2 + bne 1f + add %g0, -EFAULT, %i5 + tst %i2 + be 2f + mov %i0, %o1 + mov %i1, %o0 +5: + call __memcpy + mov %i2, %o2 + tst %o0 + bne,a 2f + add %i3, %i2, %i3 + add %i1, %i2, %i1 +2: + mov %i1, %o0 +6: + call __bzero + mov %i3, %o1 +1: + ld [%sp + 168], %o2 ! struct_ptr of parent + st %i5, [%o2] + ret + restore + + .section __ex_table,#alloc + .align 4 + .word 5b,2 + .word 6b,2 diff --git a/arch/sparc/lib/memcmp.S b/arch/sparc/lib/memcmp.S deleted file mode 100644 index cb4bdb0cc2af..000000000000 --- a/arch/sparc/lib/memcmp.S +++ /dev/null @@ -1,312 +0,0 @@ - .text - .align 4 - .global __memcmp, memcmp -__memcmp: -memcmp: -#if 1 - cmp %o2, 0 - ble L3 - mov 0, %g3 -L5: - ldub [%o0], %g2 - ldub [%o1], %g3 - sub %g2, %g3, %g2 - mov %g2, %g3 - sll %g2, 24, %g2 - - cmp %g2, 0 - bne L3 - add %o0, 1, %o0 - - add %o2, -1, %o2 - - cmp %o2, 0 - bg L5 - add %o1, 1, %o1 -L3: - sll %g3, 24, %o0 - sra %o0, 24, %o0 - - retl - nop -#else - save %sp, -104, %sp - mov %i2, %o4 - mov %i0, %o0 - - cmp %o4, 15 - ble L72 - mov %i1, %i2 - - andcc %i2, 3, %g0 - be L161 - andcc %o0, 3, %g2 -L75: - ldub [%o0], %g3 - ldub [%i2], %g2 - add %o0,1, %o0 - - subcc %g3, %g2, %i0 - bne L156 - add %i2, 1, %i2 - - andcc %i2, 3, %g0 - bne L75 - add %o4, -1, %o4 - - andcc %o0, 3, %g2 -L161: - bne,a L78 - mov %i2, %i1 - - mov %o0, %i5 - mov %i2, %i3 - srl %o4, 2, %i4 - - cmp %i4, 0 - bge L93 - mov %i4, %g2 - - add %i4, 3, %g2 -L93: - sra %g2, 2, %g2 - sll %g2, 2, %g2 - sub %i4, %g2, %g2 - - cmp %g2, 1 - be,a L88 - add %o0, 4, %i5 - - bg L94 - cmp %g2, 2 - - cmp %g2, 0 - be,a L86 - ld [%o0], %g3 - - b L162 - ld [%i5], %g3 -L94: - be L81 - cmp %g2, 3 - - be,a L83 - add %o0, -4, %i5 - - b L162 - ld [%i5], %g3 -L81: - add %o0, -8, %i5 - ld [%o0], %g3 - add %i2, -8, %i3 - ld [%i2], %g2 - - b L82 - add %i4, 2, %i4 -L83: - ld [%o0], %g4 - add %i2, -4, %i3 - ld [%i2], %g1 - - b L84 - add %i4, 1, %i4 -L86: - b L87 - ld [%i2], %g2 -L88: - add %i2, 4, %i3 - ld [%o0], %g4 - add %i4, -1, %i4 - ld [%i2], %g1 -L95: - ld [%i5], %g3 -L162: - cmp %g4, %g1 - be L87 - ld [%i3], %g2 - - cmp %g4, %g1 -L163: - bleu L114 - mov -1, %i0 - - b L114 - mov 1, %i0 -L87: - ld [%i5 + 4], %g4 - cmp %g3, %g2 - bne L163 - ld [%i3 + 4], %g1 -L84: - ld [%i5 + 8], %g3 - - cmp %g4, %g1 - bne L163 - ld [%i3 + 8], %g2 -L82: - ld [%i5 + 12], %g4 - cmp %g3, %g2 - bne L163 - ld [%i3 + 12], %g1 - - add %i5, 16, %i5 - - addcc %i4, -4, %i4 - bne L95 - add %i3, 16, %i3 - - cmp %g4, %g1 - bne L163 - nop - - b L114 - mov 0, %i0 -L78: - srl %o4, 2, %i0 - and %o0, -4, %i3 - orcc %i0, %g0, %g3 - sll %g2, 3, %o7 - mov 32, %g2 - - bge L129 - sub %g2, %o7, %o1 - - add %i0, 3, %g3 -L129: - sra %g3, 2, %g2 - sll %g2, 2, %g2 - sub %i0, %g2, %g2 - - cmp %g2, 1 - be,a L124 - ld [%i3], %o3 - - bg L130 - cmp %g2, 2 - - cmp %g2, 0 - be,a L122 - ld [%i3], %o2 - - b L164 - sll %o3, %o7, %g3 -L130: - be L117 - cmp %g2, 3 - - be,a L119 - ld [%i3], %g1 - - b L164 - sll %o3, %o7, %g3 -L117: - ld [%i3], %g4 - add %i2, -8, %i1 - ld [%i3 + 4], %o3 - add %i0, 2, %i0 - ld [%i2], %i4 - - b L118 - add %i3, -4, %i3 -L119: - ld [%i3 + 4], %g4 - add %i2, -4, %i1 - ld [%i2], %i5 - - b L120 - add %i0, 1, %i0 -L122: - ld [%i3 + 4], %g1 - ld [%i2], %i4 - - b L123 - add %i3, 4, %i3 -L124: - add %i2, 4, %i1 - ld [%i3 + 4], %o2 - add %i0, -1, %i0 - ld [%i2], %i5 - add %i3, 8, %i3 -L131: - sll %o3, %o7, %g3 -L164: - srl %o2, %o1, %g2 - ld [%i3], %g1 - or %g3, %g2, %g3 - - cmp %g3, %i5 - bne L163 - ld [%i1], %i4 -L123: - sll %o2, %o7, %g3 - srl %g1, %o1, %g2 - ld [%i3 + 4], %g4 - or %g3, %g2, %g3 - - cmp %g3, %i4 - bne L163 - ld [%i1 + 4], %i5 -L120: - sll %g1, %o7, %g3 - srl %g4, %o1, %g2 - ld [%i3 + 8], %o3 - or %g3, %g2, %g3 - - cmp %g3, %i5 - bne L163 - ld [%i1 + 8], %i4 -L118: - sll %g4, %o7, %g3 - srl %o3, %o1, %g2 - ld [%i3 + 12], %o2 - or %g3, %g2, %g3 - - cmp %g3, %i4 - bne L163 - ld [%i1 + 12], %i5 - - add %i3, 16, %i3 - addcc %i0, -4, %i0 - bne L131 - add %i1, 16, %i1 - - sll %o3, %o7, %g3 - srl %o2, %o1, %g2 - or %g3, %g2, %g3 - - cmp %g3, %i5 - be,a L114 - mov 0, %i0 - - b,a L163 -L114: - cmp %i0, 0 - bne L156 - and %o4, -4, %g2 - - add %o0, %g2, %o0 - add %i2, %g2, %i2 - and %o4, 3, %o4 -L72: - cmp %o4, 0 - be L156 - mov 0, %i0 - - ldub [%o0], %g3 -L165: - ldub [%i2], %g2 - add %o0, 1, %o0 - - subcc %g3, %g2, %i0 - bne L156 - add %i2, 1, %i2 - - addcc %o4, -1, %o4 - bne,a L165 - ldub [%o0], %g3 - - mov 0, %i0 -L156: - ret - restore -#endif diff --git a/arch/sparc/lib/memcmp_32.S b/arch/sparc/lib/memcmp_32.S new file mode 100644 index 000000000000..cb4bdb0cc2af --- /dev/null +++ b/arch/sparc/lib/memcmp_32.S @@ -0,0 +1,312 @@ + .text + .align 4 + .global __memcmp, memcmp +__memcmp: +memcmp: +#if 1 + cmp %o2, 0 + ble L3 + mov 0, %g3 +L5: + ldub [%o0], %g2 + ldub [%o1], %g3 + sub %g2, %g3, %g2 + mov %g2, %g3 + sll %g2, 24, %g2 + + cmp %g2, 0 + bne L3 + add %o0, 1, %o0 + + add %o2, -1, %o2 + + cmp %o2, 0 + bg L5 + add %o1, 1, %o1 +L3: + sll %g3, 24, %o0 + sra %o0, 24, %o0 + + retl + nop +#else + save %sp, -104, %sp + mov %i2, %o4 + mov %i0, %o0 + + cmp %o4, 15 + ble L72 + mov %i1, %i2 + + andcc %i2, 3, %g0 + be L161 + andcc %o0, 3, %g2 +L75: + ldub [%o0], %g3 + ldub [%i2], %g2 + add %o0,1, %o0 + + subcc %g3, %g2, %i0 + bne L156 + add %i2, 1, %i2 + + andcc %i2, 3, %g0 + bne L75 + add %o4, -1, %o4 + + andcc %o0, 3, %g2 +L161: + bne,a L78 + mov %i2, %i1 + + mov %o0, %i5 + mov %i2, %i3 + srl %o4, 2, %i4 + + cmp %i4, 0 + bge L93 + mov %i4, %g2 + + add %i4, 3, %g2 +L93: + sra %g2, 2, %g2 + sll %g2, 2, %g2 + sub %i4, %g2, %g2 + + cmp %g2, 1 + be,a L88 + add %o0, 4, %i5 + + bg L94 + cmp %g2, 2 + + cmp %g2, 0 + be,a L86 + ld [%o0], %g3 + + b L162 + ld [%i5], %g3 +L94: + be L81 + cmp %g2, 3 + + be,a L83 + add %o0, -4, %i5 + + b L162 + ld [%i5], %g3 +L81: + add %o0, -8, %i5 + ld [%o0], %g3 + add %i2, -8, %i3 + ld [%i2], %g2 + + b L82 + add %i4, 2, %i4 +L83: + ld [%o0], %g4 + add %i2, -4, %i3 + ld [%i2], %g1 + + b L84 + add %i4, 1, %i4 +L86: + b L87 + ld [%i2], %g2 +L88: + add %i2, 4, %i3 + ld [%o0], %g4 + add %i4, -1, %i4 + ld [%i2], %g1 +L95: + ld [%i5], %g3 +L162: + cmp %g4, %g1 + be L87 + ld [%i3], %g2 + + cmp %g4, %g1 +L163: + bleu L114 + mov -1, %i0 + + b L114 + mov 1, %i0 +L87: + ld [%i5 + 4], %g4 + cmp %g3, %g2 + bne L163 + ld [%i3 + 4], %g1 +L84: + ld [%i5 + 8], %g3 + + cmp %g4, %g1 + bne L163 + ld [%i3 + 8], %g2 +L82: + ld [%i5 + 12], %g4 + cmp %g3, %g2 + bne L163 + ld [%i3 + 12], %g1 + + add %i5, 16, %i5 + + addcc %i4, -4, %i4 + bne L95 + add %i3, 16, %i3 + + cmp %g4, %g1 + bne L163 + nop + + b L114 + mov 0, %i0 +L78: + srl %o4, 2, %i0 + and %o0, -4, %i3 + orcc %i0, %g0, %g3 + sll %g2, 3, %o7 + mov 32, %g2 + + bge L129 + sub %g2, %o7, %o1 + + add %i0, 3, %g3 +L129: + sra %g3, 2, %g2 + sll %g2, 2, %g2 + sub %i0, %g2, %g2 + + cmp %g2, 1 + be,a L124 + ld [%i3], %o3 + + bg L130 + cmp %g2, 2 + + cmp %g2, 0 + be,a L122 + ld [%i3], %o2 + + b L164 + sll %o3, %o7, %g3 +L130: + be L117 + cmp %g2, 3 + + be,a L119 + ld [%i3], %g1 + + b L164 + sll %o3, %o7, %g3 +L117: + ld [%i3], %g4 + add %i2, -8, %i1 + ld [%i3 + 4], %o3 + add %i0, 2, %i0 + ld [%i2], %i4 + + b L118 + add %i3, -4, %i3 +L119: + ld [%i3 + 4], %g4 + add %i2, -4, %i1 + ld [%i2], %i5 + + b L120 + add %i0, 1, %i0 +L122: + ld [%i3 + 4], %g1 + ld [%i2], %i4 + + b L123 + add %i3, 4, %i3 +L124: + add %i2, 4, %i1 + ld [%i3 + 4], %o2 + add %i0, -1, %i0 + ld [%i2], %i5 + add %i3, 8, %i3 +L131: + sll %o3, %o7, %g3 +L164: + srl %o2, %o1, %g2 + ld [%i3], %g1 + or %g3, %g2, %g3 + + cmp %g3, %i5 + bne L163 + ld [%i1], %i4 +L123: + sll %o2, %o7, %g3 + srl %g1, %o1, %g2 + ld [%i3 + 4], %g4 + or %g3, %g2, %g3 + + cmp %g3, %i4 + bne L163 + ld [%i1 + 4], %i5 +L120: + sll %g1, %o7, %g3 + srl %g4, %o1, %g2 + ld [%i3 + 8], %o3 + or %g3, %g2, %g3 + + cmp %g3, %i5 + bne L163 + ld [%i1 + 8], %i4 +L118: + sll %g4, %o7, %g3 + srl %o3, %o1, %g2 + ld [%i3 + 12], %o2 + or %g3, %g2, %g3 + + cmp %g3, %i4 + bne L163 + ld [%i1 + 12], %i5 + + add %i3, 16, %i3 + addcc %i0, -4, %i0 + bne L131 + add %i1, 16, %i1 + + sll %o3, %o7, %g3 + srl %o2, %o1, %g2 + or %g3, %g2, %g3 + + cmp %g3, %i5 + be,a L114 + mov 0, %i0 + + b,a L163 +L114: + cmp %i0, 0 + bne L156 + and %o4, -4, %g2 + + add %o0, %g2, %o0 + add %i2, %g2, %i2 + and %o4, 3, %o4 +L72: + cmp %o4, 0 + be L156 + mov 0, %i0 + + ldub [%o0], %g3 +L165: + ldub [%i2], %g2 + add %o0, 1, %o0 + + subcc %g3, %g2, %i0 + bne L156 + add %i2, 1, %i2 + + addcc %o4, -1, %o4 + bne,a L165 + ldub [%o0], %g3 + + mov 0, %i0 +L156: + ret + restore +#endif diff --git a/arch/sparc/lib/memscan.S b/arch/sparc/lib/memscan.S deleted file mode 100644 index 4ff1657dfc24..000000000000 --- a/arch/sparc/lib/memscan.S +++ /dev/null @@ -1,133 +0,0 @@ -/* - * memscan.S: Optimized memscan for the Sparc. - * - * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) - */ - -/* In essence, this is just a fancy strlen. */ - -#define LO_MAGIC 0x01010101 -#define HI_MAGIC 0x80808080 - - .text - .align 4 - .globl __memscan_zero, __memscan_generic - .globl memscan -__memscan_zero: - /* %o0 = addr, %o1 = size */ - cmp %o1, 0 - bne,a 1f - andcc %o0, 3, %g0 - - retl - nop - -1: - be mzero_scan_word - sethi %hi(HI_MAGIC), %g2 - - ldsb [%o0], %g3 -mzero_still_not_word_aligned: - cmp %g3, 0 - bne 1f - add %o0, 1, %o0 - - retl - sub %o0, 1, %o0 - -1: - subcc %o1, 1, %o1 - bne,a 1f - andcc %o0, 3, %g0 - - retl - nop - -1: - bne,a mzero_still_not_word_aligned - ldsb [%o0], %g3 - - sethi %hi(HI_MAGIC), %g2 -mzero_scan_word: - or %g2, %lo(HI_MAGIC), %o3 - sethi %hi(LO_MAGIC), %g3 - or %g3, %lo(LO_MAGIC), %o2 -mzero_next_word: - ld [%o0], %g2 -mzero_next_word_preloaded: - sub %g2, %o2, %g2 -mzero_next_word_preloaded_next: - andcc %g2, %o3, %g0 - bne mzero_byte_zero - add %o0, 4, %o0 - -mzero_check_out_of_fuel: - subcc %o1, 4, %o1 - bg,a 1f - ld [%o0], %g2 - - retl - nop - -1: - b mzero_next_word_preloaded_next - sub %g2, %o2, %g2 - - /* Check every byte. */ -mzero_byte_zero: - ldsb [%o0 - 4], %g2 - cmp %g2, 0 - bne mzero_byte_one - sub %o0, 4, %g3 - - retl - mov %g3, %o0 - -mzero_byte_one: - ldsb [%o0 - 3], %g2 - cmp %g2, 0 - bne,a mzero_byte_two_and_three - ldsb [%o0 - 2], %g2 - - retl - sub %o0, 3, %o0 - -mzero_byte_two_and_three: - cmp %g2, 0 - bne,a 1f - ldsb [%o0 - 1], %g2 - - retl - sub %o0, 2, %o0 - -1: - cmp %g2, 0 - bne,a mzero_next_word_preloaded - ld [%o0], %g2 - - retl - sub %o0, 1, %o0 - -mzero_found_it: - retl - sub %o0, 2, %o0 - -memscan: -__memscan_generic: - /* %o0 = addr, %o1 = c, %o2 = size */ - cmp %o2, 0 - bne,a 0f - ldub [%o0], %g2 - - b,a 2f -1: - ldub [%o0], %g2 -0: - cmp %g2, %o1 - be 2f - addcc %o2, -1, %o2 - bne 1b - add %o0, 1, %o0 -2: - retl - nop diff --git a/arch/sparc/lib/memscan_32.S b/arch/sparc/lib/memscan_32.S new file mode 100644 index 000000000000..4ff1657dfc24 --- /dev/null +++ b/arch/sparc/lib/memscan_32.S @@ -0,0 +1,133 @@ +/* + * memscan.S: Optimized memscan for the Sparc. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + */ + +/* In essence, this is just a fancy strlen. */ + +#define LO_MAGIC 0x01010101 +#define HI_MAGIC 0x80808080 + + .text + .align 4 + .globl __memscan_zero, __memscan_generic + .globl memscan +__memscan_zero: + /* %o0 = addr, %o1 = size */ + cmp %o1, 0 + bne,a 1f + andcc %o0, 3, %g0 + + retl + nop + +1: + be mzero_scan_word + sethi %hi(HI_MAGIC), %g2 + + ldsb [%o0], %g3 +mzero_still_not_word_aligned: + cmp %g3, 0 + bne 1f + add %o0, 1, %o0 + + retl + sub %o0, 1, %o0 + +1: + subcc %o1, 1, %o1 + bne,a 1f + andcc %o0, 3, %g0 + + retl + nop + +1: + bne,a mzero_still_not_word_aligned + ldsb [%o0], %g3 + + sethi %hi(HI_MAGIC), %g2 +mzero_scan_word: + or %g2, %lo(HI_MAGIC), %o3 + sethi %hi(LO_MAGIC), %g3 + or %g3, %lo(LO_MAGIC), %o2 +mzero_next_word: + ld [%o0], %g2 +mzero_next_word_preloaded: + sub %g2, %o2, %g2 +mzero_next_word_preloaded_next: + andcc %g2, %o3, %g0 + bne mzero_byte_zero + add %o0, 4, %o0 + +mzero_check_out_of_fuel: + subcc %o1, 4, %o1 + bg,a 1f + ld [%o0], %g2 + + retl + nop + +1: + b mzero_next_word_preloaded_next + sub %g2, %o2, %g2 + + /* Check every byte. */ +mzero_byte_zero: + ldsb [%o0 - 4], %g2 + cmp %g2, 0 + bne mzero_byte_one + sub %o0, 4, %g3 + + retl + mov %g3, %o0 + +mzero_byte_one: + ldsb [%o0 - 3], %g2 + cmp %g2, 0 + bne,a mzero_byte_two_and_three + ldsb [%o0 - 2], %g2 + + retl + sub %o0, 3, %o0 + +mzero_byte_two_and_three: + cmp %g2, 0 + bne,a 1f + ldsb [%o0 - 1], %g2 + + retl + sub %o0, 2, %o0 + +1: + cmp %g2, 0 + bne,a mzero_next_word_preloaded + ld [%o0], %g2 + + retl + sub %o0, 1, %o0 + +mzero_found_it: + retl + sub %o0, 2, %o0 + +memscan: +__memscan_generic: + /* %o0 = addr, %o1 = c, %o2 = size */ + cmp %o2, 0 + bne,a 0f + ldub [%o0], %g2 + + b,a 2f +1: + ldub [%o0], %g2 +0: + cmp %g2, %o1 + be 2f + addcc %o2, -1, %o2 + bne 1b + add %o0, 1, %o0 +2: + retl + nop diff --git a/arch/sparc/lib/rwsem.S b/arch/sparc/lib/rwsem.S deleted file mode 100644 index 9675268e7fde..000000000000 --- a/arch/sparc/lib/rwsem.S +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Assembly part of rw semaphores. - * - * Copyright (C) 1999 Jakub Jelinek (jakub@redhat.com) - */ - -#include -#include - - .section .sched.text, "ax" - .align 4 - - .globl ___down_read -___down_read: - rd %psr, %g3 - nop - nop - nop - or %g3, PSR_PIL, %g7 - wr %g7, 0, %psr - nop - nop - nop -#ifdef CONFIG_SMP -1: ldstub [%g1 + 4], %g7 - tst %g7 - bne 1b - ld [%g1], %g7 - sub %g7, 1, %g7 - st %g7, [%g1] - stb %g0, [%g1 + 4] -#else - ld [%g1], %g7 - sub %g7, 1, %g7 - st %g7, [%g1] -#endif - wr %g3, 0, %psr - add %g7, 1, %g7 - nop - nop - subcc %g7, 1, %g7 - bneg 3f - nop -2: jmpl %o7, %g0 - mov %g4, %o7 -3: save %sp, -64, %sp - mov %g1, %l1 - mov %g4, %l4 - bcs 4f - mov %g5, %l5 - call down_read_failed - mov %l1, %o0 - mov %l1, %g1 - mov %l4, %g4 - ba ___down_read - restore %l5, %g0, %g5 -4: call down_read_failed_biased - mov %l1, %o0 - mov %l1, %g1 - mov %l4, %g4 - ba 2b - restore %l5, %g0, %g5 - - .globl ___down_write -___down_write: - rd %psr, %g3 - nop - nop - nop - or %g3, PSR_PIL, %g7 - wr %g7, 0, %psr - sethi %hi(0x01000000), %g2 - nop - nop -#ifdef CONFIG_SMP -1: ldstub [%g1 + 4], %g7 - tst %g7 - bne 1b - ld [%g1], %g7 - sub %g7, %g2, %g7 - st %g7, [%g1] - stb %g0, [%g1 + 4] -#else - ld [%g1], %g7 - sub %g7, %g2, %g7 - st %g7, [%g1] -#endif - wr %g3, 0, %psr - add %g7, %g2, %g7 - nop - nop - subcc %g7, %g2, %g7 - bne 3f - nop -2: jmpl %o7, %g0 - mov %g4, %o7 -3: save %sp, -64, %sp - mov %g1, %l1 - mov %g4, %l4 - bcs 4f - mov %g5, %l5 - call down_write_failed - mov %l1, %o0 - mov %l1, %g1 - mov %l4, %g4 - ba ___down_write - restore %l5, %g0, %g5 -4: call down_write_failed_biased - mov %l1, %o0 - mov %l1, %g1 - mov %l4, %g4 - ba 2b - restore %l5, %g0, %g5 - - .text - .globl ___up_read -___up_read: - rd %psr, %g3 - nop - nop - nop - or %g3, PSR_PIL, %g7 - wr %g7, 0, %psr - nop - nop - nop -#ifdef CONFIG_SMP -1: ldstub [%g1 + 4], %g7 - tst %g7 - bne 1b - ld [%g1], %g7 - add %g7, 1, %g7 - st %g7, [%g1] - stb %g0, [%g1 + 4] -#else - ld [%g1], %g7 - add %g7, 1, %g7 - st %g7, [%g1] -#endif - wr %g3, 0, %psr - nop - nop - nop - cmp %g7, 0 - be 3f - nop -2: jmpl %o7, %g0 - mov %g4, %o7 -3: save %sp, -64, %sp - mov %g1, %l1 - mov %g4, %l4 - mov %g5, %l5 - clr %o1 - call __rwsem_wake - mov %l1, %o0 - mov %l1, %g1 - mov %l4, %g4 - ba 2b - restore %l5, %g0, %g5 - - .globl ___up_write -___up_write: - rd %psr, %g3 - nop - nop - nop - or %g3, PSR_PIL, %g7 - wr %g7, 0, %psr - sethi %hi(0x01000000), %g2 - nop - nop -#ifdef CONFIG_SMP -1: ldstub [%g1 + 4], %g7 - tst %g7 - bne 1b - ld [%g1], %g7 - add %g7, %g2, %g7 - st %g7, [%g1] - stb %g0, [%g1 + 4] -#else - ld [%g1], %g7 - add %g7, %g2, %g7 - st %g7, [%g1] -#endif - wr %g3, 0, %psr - sub %g7, %g2, %g7 - nop - nop - addcc %g7, %g2, %g7 - bcs 3f - nop -2: jmpl %o7, %g0 - mov %g4, %o7 -3: save %sp, -64, %sp - mov %g1, %l1 - mov %g4, %l4 - mov %g5, %l5 - mov %g7, %o1 - call __rwsem_wake - mov %l1, %o0 - mov %l1, %g1 - mov %l4, %g4 - ba 2b - restore %l5, %g0, %g5 diff --git a/arch/sparc/lib/rwsem_32.S b/arch/sparc/lib/rwsem_32.S new file mode 100644 index 000000000000..9675268e7fde --- /dev/null +++ b/arch/sparc/lib/rwsem_32.S @@ -0,0 +1,204 @@ +/* + * Assembly part of rw semaphores. + * + * Copyright (C) 1999 Jakub Jelinek (jakub@redhat.com) + */ + +#include +#include + + .section .sched.text, "ax" + .align 4 + + .globl ___down_read +___down_read: + rd %psr, %g3 + nop + nop + nop + or %g3, PSR_PIL, %g7 + wr %g7, 0, %psr + nop + nop + nop +#ifdef CONFIG_SMP +1: ldstub [%g1 + 4], %g7 + tst %g7 + bne 1b + ld [%g1], %g7 + sub %g7, 1, %g7 + st %g7, [%g1] + stb %g0, [%g1 + 4] +#else + ld [%g1], %g7 + sub %g7, 1, %g7 + st %g7, [%g1] +#endif + wr %g3, 0, %psr + add %g7, 1, %g7 + nop + nop + subcc %g7, 1, %g7 + bneg 3f + nop +2: jmpl %o7, %g0 + mov %g4, %o7 +3: save %sp, -64, %sp + mov %g1, %l1 + mov %g4, %l4 + bcs 4f + mov %g5, %l5 + call down_read_failed + mov %l1, %o0 + mov %l1, %g1 + mov %l4, %g4 + ba ___down_read + restore %l5, %g0, %g5 +4: call down_read_failed_biased + mov %l1, %o0 + mov %l1, %g1 + mov %l4, %g4 + ba 2b + restore %l5, %g0, %g5 + + .globl ___down_write +___down_write: + rd %psr, %g3 + nop + nop + nop + or %g3, PSR_PIL, %g7 + wr %g7, 0, %psr + sethi %hi(0x01000000), %g2 + nop + nop +#ifdef CONFIG_SMP +1: ldstub [%g1 + 4], %g7 + tst %g7 + bne 1b + ld [%g1], %g7 + sub %g7, %g2, %g7 + st %g7, [%g1] + stb %g0, [%g1 + 4] +#else + ld [%g1], %g7 + sub %g7, %g2, %g7 + st %g7, [%g1] +#endif + wr %g3, 0, %psr + add %g7, %g2, %g7 + nop + nop + subcc %g7, %g2, %g7 + bne 3f + nop +2: jmpl %o7, %g0 + mov %g4, %o7 +3: save %sp, -64, %sp + mov %g1, %l1 + mov %g4, %l4 + bcs 4f + mov %g5, %l5 + call down_write_failed + mov %l1, %o0 + mov %l1, %g1 + mov %l4, %g4 + ba ___down_write + restore %l5, %g0, %g5 +4: call down_write_failed_biased + mov %l1, %o0 + mov %l1, %g1 + mov %l4, %g4 + ba 2b + restore %l5, %g0, %g5 + + .text + .globl ___up_read +___up_read: + rd %psr, %g3 + nop + nop + nop + or %g3, PSR_PIL, %g7 + wr %g7, 0, %psr + nop + nop + nop +#ifdef CONFIG_SMP +1: ldstub [%g1 + 4], %g7 + tst %g7 + bne 1b + ld [%g1], %g7 + add %g7, 1, %g7 + st %g7, [%g1] + stb %g0, [%g1 + 4] +#else + ld [%g1], %g7 + add %g7, 1, %g7 + st %g7, [%g1] +#endif + wr %g3, 0, %psr + nop + nop + nop + cmp %g7, 0 + be 3f + nop +2: jmpl %o7, %g0 + mov %g4, %o7 +3: save %sp, -64, %sp + mov %g1, %l1 + mov %g4, %l4 + mov %g5, %l5 + clr %o1 + call __rwsem_wake + mov %l1, %o0 + mov %l1, %g1 + mov %l4, %g4 + ba 2b + restore %l5, %g0, %g5 + + .globl ___up_write +___up_write: + rd %psr, %g3 + nop + nop + nop + or %g3, PSR_PIL, %g7 + wr %g7, 0, %psr + sethi %hi(0x01000000), %g2 + nop + nop +#ifdef CONFIG_SMP +1: ldstub [%g1 + 4], %g7 + tst %g7 + bne 1b + ld [%g1], %g7 + add %g7, %g2, %g7 + st %g7, [%g1] + stb %g0, [%g1 + 4] +#else + ld [%g1], %g7 + add %g7, %g2, %g7 + st %g7, [%g1] +#endif + wr %g3, 0, %psr + sub %g7, %g2, %g7 + nop + nop + addcc %g7, %g2, %g7 + bcs 3f + nop +2: jmpl %o7, %g0 + mov %g4, %o7 +3: save %sp, -64, %sp + mov %g1, %l1 + mov %g4, %l4 + mov %g5, %l5 + mov %g7, %o1 + call __rwsem_wake + mov %l1, %o0 + mov %l1, %g1 + mov %l4, %g4 + ba 2b + restore %l5, %g0, %g5 diff --git a/arch/sparc/lib/strlen.S b/arch/sparc/lib/strlen.S deleted file mode 100644 index ed9a763368cd..000000000000 --- a/arch/sparc/lib/strlen.S +++ /dev/null @@ -1,81 +0,0 @@ -/* strlen.S: Sparc optimized strlen code - * Hand optimized from GNU libc's strlen - * Copyright (C) 1991,1996 Free Software Foundation - * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) - * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#define LO_MAGIC 0x01010101 -#define HI_MAGIC 0x80808080 - -0: - ldub [%o0], %o5 - cmp %o5, 0 - be 1f - add %o0, 1, %o0 - andcc %o0, 3, %g0 - be 4f - or %o4, %lo(HI_MAGIC), %o3 - ldub [%o0], %o5 - cmp %o5, 0 - be 2f - add %o0, 1, %o0 - andcc %o0, 3, %g0 - be 5f - sethi %hi(LO_MAGIC), %o4 - ldub [%o0], %o5 - cmp %o5, 0 - be 3f - add %o0, 1, %o0 - b 8f - or %o4, %lo(LO_MAGIC), %o2 -1: - retl - mov 0, %o0 -2: - retl - mov 1, %o0 -3: - retl - mov 2, %o0 - - .align 4 - .global strlen -strlen: - mov %o0, %o1 - andcc %o0, 3, %g0 - bne 0b - sethi %hi(HI_MAGIC), %o4 - or %o4, %lo(HI_MAGIC), %o3 -4: - sethi %hi(LO_MAGIC), %o4 -5: - or %o4, %lo(LO_MAGIC), %o2 -8: - ld [%o0], %o5 -2: - sub %o5, %o2, %o4 - andcc %o4, %o3, %g0 - be 8b - add %o0, 4, %o0 - - /* Check every byte. */ - srl %o5, 24, %g5 - andcc %g5, 0xff, %g0 - be 1f - add %o0, -4, %o4 - srl %o5, 16, %g5 - andcc %g5, 0xff, %g0 - be 1f - add %o4, 1, %o4 - srl %o5, 8, %g5 - andcc %g5, 0xff, %g0 - be 1f - add %o4, 1, %o4 - andcc %o5, 0xff, %g0 - bne,a 2b - ld [%o0], %o5 - add %o4, 1, %o4 -1: - retl - sub %o4, %o1, %o0 diff --git a/arch/sparc/lib/strlen_32.S b/arch/sparc/lib/strlen_32.S new file mode 100644 index 000000000000..ed9a763368cd --- /dev/null +++ b/arch/sparc/lib/strlen_32.S @@ -0,0 +1,81 @@ +/* strlen.S: Sparc optimized strlen code + * Hand optimized from GNU libc's strlen + * Copyright (C) 1991,1996 Free Software Foundation + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#define LO_MAGIC 0x01010101 +#define HI_MAGIC 0x80808080 + +0: + ldub [%o0], %o5 + cmp %o5, 0 + be 1f + add %o0, 1, %o0 + andcc %o0, 3, %g0 + be 4f + or %o4, %lo(HI_MAGIC), %o3 + ldub [%o0], %o5 + cmp %o5, 0 + be 2f + add %o0, 1, %o0 + andcc %o0, 3, %g0 + be 5f + sethi %hi(LO_MAGIC), %o4 + ldub [%o0], %o5 + cmp %o5, 0 + be 3f + add %o0, 1, %o0 + b 8f + or %o4, %lo(LO_MAGIC), %o2 +1: + retl + mov 0, %o0 +2: + retl + mov 1, %o0 +3: + retl + mov 2, %o0 + + .align 4 + .global strlen +strlen: + mov %o0, %o1 + andcc %o0, 3, %g0 + bne 0b + sethi %hi(HI_MAGIC), %o4 + or %o4, %lo(HI_MAGIC), %o3 +4: + sethi %hi(LO_MAGIC), %o4 +5: + or %o4, %lo(LO_MAGIC), %o2 +8: + ld [%o0], %o5 +2: + sub %o5, %o2, %o4 + andcc %o4, %o3, %g0 + be 8b + add %o0, 4, %o0 + + /* Check every byte. */ + srl %o5, 24, %g5 + andcc %g5, 0xff, %g0 + be 1f + add %o0, -4, %o4 + srl %o5, 16, %g5 + andcc %g5, 0xff, %g0 + be 1f + add %o4, 1, %o4 + srl %o5, 8, %g5 + andcc %g5, 0xff, %g0 + be 1f + add %o4, 1, %o4 + andcc %o5, 0xff, %g0 + bne,a 2b + ld [%o0], %o5 + add %o4, 1, %o4 +1: + retl + sub %o4, %o1, %o0 diff --git a/arch/sparc/lib/strlen_user.S b/arch/sparc/lib/strlen_user.S deleted file mode 100644 index 8c8a371df3c9..000000000000 --- a/arch/sparc/lib/strlen_user.S +++ /dev/null @@ -1,109 +0,0 @@ -/* strlen_user.S: Sparc optimized strlen_user code - * - * Return length of string in userspace including terminating 0 - * or 0 for error - * - * Copyright (C) 1991,1996 Free Software Foundation - * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) - * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#define LO_MAGIC 0x01010101 -#define HI_MAGIC 0x80808080 - -10: - ldub [%o0], %o5 - cmp %o5, 0 - be 1f - add %o0, 1, %o0 - andcc %o0, 3, %g0 - be 4f - or %o4, %lo(HI_MAGIC), %o3 -11: - ldub [%o0], %o5 - cmp %o5, 0 - be 2f - add %o0, 1, %o0 - andcc %o0, 3, %g0 - be 5f - sethi %hi(LO_MAGIC), %o4 -12: - ldub [%o0], %o5 - cmp %o5, 0 - be 3f - add %o0, 1, %o0 - b 13f - or %o4, %lo(LO_MAGIC), %o2 -1: - retl - mov 1, %o0 -2: - retl - mov 2, %o0 -3: - retl - mov 3, %o0 - - .align 4 - .global __strlen_user, __strnlen_user -__strlen_user: - sethi %hi(32768), %o1 -__strnlen_user: - mov %o1, %g1 - mov %o0, %o1 - andcc %o0, 3, %g0 - bne 10b - sethi %hi(HI_MAGIC), %o4 - or %o4, %lo(HI_MAGIC), %o3 -4: - sethi %hi(LO_MAGIC), %o4 -5: - or %o4, %lo(LO_MAGIC), %o2 -13: - ld [%o0], %o5 -2: - sub %o5, %o2, %o4 - andcc %o4, %o3, %g0 - bne 82f - add %o0, 4, %o0 - sub %o0, %o1, %g2 -81: cmp %g2, %g1 - blu 13b - mov %o0, %o4 - ba,a 1f - - /* Check every byte. */ -82: srl %o5, 24, %g5 - andcc %g5, 0xff, %g0 - be 1f - add %o0, -3, %o4 - srl %o5, 16, %g5 - andcc %g5, 0xff, %g0 - be 1f - add %o4, 1, %o4 - srl %o5, 8, %g5 - andcc %g5, 0xff, %g0 - be 1f - add %o4, 1, %o4 - andcc %o5, 0xff, %g0 - bne 81b - sub %o0, %o1, %g2 - - add %o4, 1, %o4 -1: - retl - sub %o4, %o1, %o0 - - .section .fixup,#alloc,#execinstr - .align 4 -9: - retl - clr %o0 - - .section __ex_table,#alloc - .align 4 - - .word 10b, 9b - .word 11b, 9b - .word 12b, 9b - .word 13b, 9b diff --git a/arch/sparc/lib/strlen_user_32.S b/arch/sparc/lib/strlen_user_32.S new file mode 100644 index 000000000000..8c8a371df3c9 --- /dev/null +++ b/arch/sparc/lib/strlen_user_32.S @@ -0,0 +1,109 @@ +/* strlen_user.S: Sparc optimized strlen_user code + * + * Return length of string in userspace including terminating 0 + * or 0 for error + * + * Copyright (C) 1991,1996 Free Software Foundation + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#define LO_MAGIC 0x01010101 +#define HI_MAGIC 0x80808080 + +10: + ldub [%o0], %o5 + cmp %o5, 0 + be 1f + add %o0, 1, %o0 + andcc %o0, 3, %g0 + be 4f + or %o4, %lo(HI_MAGIC), %o3 +11: + ldub [%o0], %o5 + cmp %o5, 0 + be 2f + add %o0, 1, %o0 + andcc %o0, 3, %g0 + be 5f + sethi %hi(LO_MAGIC), %o4 +12: + ldub [%o0], %o5 + cmp %o5, 0 + be 3f + add %o0, 1, %o0 + b 13f + or %o4, %lo(LO_MAGIC), %o2 +1: + retl + mov 1, %o0 +2: + retl + mov 2, %o0 +3: + retl + mov 3, %o0 + + .align 4 + .global __strlen_user, __strnlen_user +__strlen_user: + sethi %hi(32768), %o1 +__strnlen_user: + mov %o1, %g1 + mov %o0, %o1 + andcc %o0, 3, %g0 + bne 10b + sethi %hi(HI_MAGIC), %o4 + or %o4, %lo(HI_MAGIC), %o3 +4: + sethi %hi(LO_MAGIC), %o4 +5: + or %o4, %lo(LO_MAGIC), %o2 +13: + ld [%o0], %o5 +2: + sub %o5, %o2, %o4 + andcc %o4, %o3, %g0 + bne 82f + add %o0, 4, %o0 + sub %o0, %o1, %g2 +81: cmp %g2, %g1 + blu 13b + mov %o0, %o4 + ba,a 1f + + /* Check every byte. */ +82: srl %o5, 24, %g5 + andcc %g5, 0xff, %g0 + be 1f + add %o0, -3, %o4 + srl %o5, 16, %g5 + andcc %g5, 0xff, %g0 + be 1f + add %o4, 1, %o4 + srl %o5, 8, %g5 + andcc %g5, 0xff, %g0 + be 1f + add %o4, 1, %o4 + andcc %o5, 0xff, %g0 + bne 81b + sub %o0, %o1, %g2 + + add %o4, 1, %o4 +1: + retl + sub %o4, %o1, %o0 + + .section .fixup,#alloc,#execinstr + .align 4 +9: + retl + clr %o0 + + .section __ex_table,#alloc + .align 4 + + .word 10b, 9b + .word 11b, 9b + .word 12b, 9b + .word 13b, 9b diff --git a/arch/sparc/lib/strncmp.S b/arch/sparc/lib/strncmp.S deleted file mode 100644 index 494ec664537a..000000000000 --- a/arch/sparc/lib/strncmp.S +++ /dev/null @@ -1,118 +0,0 @@ -/* - * strncmp.S: Hand optimized Sparc assembly of GCC output from GNU libc - * generic strncmp routine. - */ - - .text - .align 4 - .global __strncmp, strncmp -__strncmp: -strncmp: - mov %o0, %g3 - mov 0, %o3 - - cmp %o2, 3 - ble 7f - mov 0, %g2 - - sra %o2, 2, %o4 - ldub [%g3], %o3 - -0: - ldub [%o1], %g2 - add %g3, 1, %g3 - and %o3, 0xff, %o0 - - cmp %o0, 0 - be 8f - add %o1, 1, %o1 - - cmp %o0, %g2 - be,a 1f - ldub [%g3], %o3 - - retl - sub %o0, %g2, %o0 - -1: - ldub [%o1], %g2 - add %g3,1, %g3 - and %o3, 0xff, %o0 - - cmp %o0, 0 - be 8f - add %o1, 1, %o1 - - cmp %o0, %g2 - be,a 1f - ldub [%g3], %o3 - - retl - sub %o0, %g2, %o0 - -1: - ldub [%o1], %g2 - add %g3, 1, %g3 - and %o3, 0xff, %o0 - - cmp %o0, 0 - be 8f - add %o1, 1, %o1 - - cmp %o0, %g2 - be,a 1f - ldub [%g3], %o3 - - retl - sub %o0, %g2, %o0 - -1: - ldub [%o1], %g2 - add %g3, 1, %g3 - and %o3, 0xff, %o0 - - cmp %o0, 0 - be 8f - add %o1, 1, %o1 - - cmp %o0, %g2 - be 1f - add %o4, -1, %o4 - - retl - sub %o0, %g2, %o0 - -1: - - cmp %o4, 0 - bg,a 0b - ldub [%g3], %o3 - - b 7f - and %o2, 3, %o2 - -9: - ldub [%o1], %g2 - add %g3, 1, %g3 - and %o3, 0xff, %o0 - - cmp %o0, 0 - be 8f - add %o1, 1, %o1 - - cmp %o0, %g2 - be 7f - add %o2, -1, %o2 - -8: - retl - sub %o0, %g2, %o0 - -7: - cmp %o2, 0 - bg,a 9b - ldub [%g3], %o3 - - and %g2, 0xff, %o0 - retl - sub %o3, %o0, %o0 diff --git a/arch/sparc/lib/strncmp_32.S b/arch/sparc/lib/strncmp_32.S new file mode 100644 index 000000000000..494ec664537a --- /dev/null +++ b/arch/sparc/lib/strncmp_32.S @@ -0,0 +1,118 @@ +/* + * strncmp.S: Hand optimized Sparc assembly of GCC output from GNU libc + * generic strncmp routine. + */ + + .text + .align 4 + .global __strncmp, strncmp +__strncmp: +strncmp: + mov %o0, %g3 + mov 0, %o3 + + cmp %o2, 3 + ble 7f + mov 0, %g2 + + sra %o2, 2, %o4 + ldub [%g3], %o3 + +0: + ldub [%o1], %g2 + add %g3, 1, %g3 + and %o3, 0xff, %o0 + + cmp %o0, 0 + be 8f + add %o1, 1, %o1 + + cmp %o0, %g2 + be,a 1f + ldub [%g3], %o3 + + retl + sub %o0, %g2, %o0 + +1: + ldub [%o1], %g2 + add %g3,1, %g3 + and %o3, 0xff, %o0 + + cmp %o0, 0 + be 8f + add %o1, 1, %o1 + + cmp %o0, %g2 + be,a 1f + ldub [%g3], %o3 + + retl + sub %o0, %g2, %o0 + +1: + ldub [%o1], %g2 + add %g3, 1, %g3 + and %o3, 0xff, %o0 + + cmp %o0, 0 + be 8f + add %o1, 1, %o1 + + cmp %o0, %g2 + be,a 1f + ldub [%g3], %o3 + + retl + sub %o0, %g2, %o0 + +1: + ldub [%o1], %g2 + add %g3, 1, %g3 + and %o3, 0xff, %o0 + + cmp %o0, 0 + be 8f + add %o1, 1, %o1 + + cmp %o0, %g2 + be 1f + add %o4, -1, %o4 + + retl + sub %o0, %g2, %o0 + +1: + + cmp %o4, 0 + bg,a 0b + ldub [%g3], %o3 + + b 7f + and %o2, 3, %o2 + +9: + ldub [%o1], %g2 + add %g3, 1, %g3 + and %o3, 0xff, %o0 + + cmp %o0, 0 + be 8f + add %o1, 1, %o1 + + cmp %o0, %g2 + be 7f + add %o2, -1, %o2 + +8: + retl + sub %o0, %g2, %o0 + +7: + cmp %o2, 0 + bg,a 9b + ldub [%g3], %o3 + + and %g2, 0xff, %o0 + retl + sub %o3, %o0, %o0 diff --git a/arch/sparc/lib/strncpy_from_user.S b/arch/sparc/lib/strncpy_from_user.S deleted file mode 100644 index d77198976a66..000000000000 --- a/arch/sparc/lib/strncpy_from_user.S +++ /dev/null @@ -1,47 +0,0 @@ -/* strncpy_from_user.S: Sparc strncpy from userspace. - * - * Copyright(C) 1996 David S. Miller - */ - -#include -#include - - .text - .align 4 - - /* Must return: - * - * -EFAULT for an exception - * count if we hit the buffer limit - * bytes copied if we hit a null byte - */ - - .globl __strncpy_from_user -__strncpy_from_user: - /* %o0=dest, %o1=src, %o2=count */ - mov %o2, %o3 -1: - subcc %o2, 1, %o2 - bneg 2f - nop -10: - ldub [%o1], %o4 - add %o0, 1, %o0 - cmp %o4, 0 - add %o1, 1, %o1 - bne 1b - stb %o4, [%o0 - 1] -2: - add %o2, 1, %o0 - retl - sub %o3, %o0, %o0 - - .section .fixup,#alloc,#execinstr - .align 4 -4: - retl - mov -EFAULT, %o0 - - .section __ex_table,#alloc - .align 4 - .word 10b, 4b diff --git a/arch/sparc/lib/strncpy_from_user_32.S b/arch/sparc/lib/strncpy_from_user_32.S new file mode 100644 index 000000000000..d77198976a66 --- /dev/null +++ b/arch/sparc/lib/strncpy_from_user_32.S @@ -0,0 +1,47 @@ +/* strncpy_from_user.S: Sparc strncpy from userspace. + * + * Copyright(C) 1996 David S. Miller + */ + +#include +#include + + .text + .align 4 + + /* Must return: + * + * -EFAULT for an exception + * count if we hit the buffer limit + * bytes copied if we hit a null byte + */ + + .globl __strncpy_from_user +__strncpy_from_user: + /* %o0=dest, %o1=src, %o2=count */ + mov %o2, %o3 +1: + subcc %o2, 1, %o2 + bneg 2f + nop +10: + ldub [%o1], %o4 + add %o0, 1, %o0 + cmp %o4, 0 + add %o1, 1, %o1 + bne 1b + stb %o4, [%o0 - 1] +2: + add %o2, 1, %o0 + retl + sub %o3, %o0, %o0 + + .section .fixup,#alloc,#execinstr + .align 4 +4: + retl + mov -EFAULT, %o0 + + .section __ex_table,#alloc + .align 4 + .word 10b, 4b -- cgit v1.2.3 From 478b8fecda511942404ac232897a718cecd13e48 Mon Sep 17 00:00:00 2001 From: Sam Ravnborg Date: Wed, 3 Dec 2008 03:10:25 -0800 Subject: sparc,sparc64: unify lib/ o Renamed files in sparc64 to _64.S when identical to sparc32 files. o iomap.c were equal for sparc32 and sparc64 o adjusted sparc/Makefile now we have only one lib/ Signed-off-by: Sam Ravnborg Signed-off-by: David S. Miller --- arch/sparc/lib/GENbzero.S | 160 +++++++++ arch/sparc/lib/GENcopy_from_user.S | 34 ++ arch/sparc/lib/GENcopy_to_user.S | 38 ++ arch/sparc/lib/GENmemcpy.S | 121 +++++++ arch/sparc/lib/GENpage.S | 77 ++++ arch/sparc/lib/GENpatch.S | 33 ++ arch/sparc/lib/Makefile | 22 ++ arch/sparc/lib/NG2copy_from_user.S | 40 +++ arch/sparc/lib/NG2copy_to_user.S | 49 +++ arch/sparc/lib/NG2memcpy.S | 520 +++++++++++++++++++++++++++ arch/sparc/lib/NG2page.S | 61 ++++ arch/sparc/lib/NG2patch.S | 33 ++ arch/sparc/lib/NGbzero.S | 164 +++++++++ arch/sparc/lib/NGcopy_from_user.S | 37 ++ arch/sparc/lib/NGcopy_to_user.S | 40 +++ arch/sparc/lib/NGmemcpy.S | 425 ++++++++++++++++++++++ arch/sparc/lib/NGpage.S | 99 ++++++ arch/sparc/lib/NGpatch.S | 33 ++ arch/sparc/lib/PeeCeeI.c | 203 +++++++++++ arch/sparc/lib/U1copy_from_user.S | 33 ++ arch/sparc/lib/U1copy_to_user.S | 33 ++ arch/sparc/lib/U1memcpy.S | 563 +++++++++++++++++++++++++++++ arch/sparc/lib/U3copy_from_user.S | 22 ++ arch/sparc/lib/U3copy_to_user.S | 33 ++ arch/sparc/lib/U3memcpy.S | 422 ++++++++++++++++++++++ arch/sparc/lib/U3patch.S | 33 ++ arch/sparc/lib/VISsave.S | 144 ++++++++ arch/sparc/lib/atomic_64.S | 138 +++++++ arch/sparc/lib/bitops.S | 141 ++++++++ arch/sparc/lib/bzero.S | 158 ++++++++ arch/sparc/lib/checksum_64.S | 173 +++++++++ arch/sparc/lib/clear_page.S | 103 ++++++ arch/sparc/lib/copy_in_user.S | 119 +++++++ arch/sparc/lib/copy_page.S | 250 +++++++++++++ arch/sparc/lib/csum_copy.S | 309 ++++++++++++++++ arch/sparc/lib/csum_copy_from_user.S | 21 ++ arch/sparc/lib/csum_copy_to_user.S | 21 ++ arch/sparc/lib/ipcsum.S | 34 ++ arch/sparc/lib/mcount.S | 143 ++++++++ arch/sparc/lib/memcmp_64.S | 28 ++ arch/sparc/lib/memmove.S | 31 ++ arch/sparc/lib/memscan_64.S | 129 +++++++ arch/sparc/lib/rwsem_64.S | 163 +++++++++ arch/sparc/lib/strlen_64.S | 80 +++++ arch/sparc/lib/strlen_user_64.S | 95 +++++ arch/sparc/lib/strncmp_64.S | 32 ++ arch/sparc/lib/strncpy_from_user_64.S | 135 +++++++ arch/sparc/lib/user_fixup.c | 66 ++++ arch/sparc/lib/xor.S | 652 ++++++++++++++++++++++++++++++++++ 49 files changed, 6493 insertions(+) create mode 100644 arch/sparc/lib/GENbzero.S create mode 100644 arch/sparc/lib/GENcopy_from_user.S create mode 100644 arch/sparc/lib/GENcopy_to_user.S create mode 100644 arch/sparc/lib/GENmemcpy.S create mode 100644 arch/sparc/lib/GENpage.S create mode 100644 arch/sparc/lib/GENpatch.S create mode 100644 arch/sparc/lib/NG2copy_from_user.S create mode 100644 arch/sparc/lib/NG2copy_to_user.S create mode 100644 arch/sparc/lib/NG2memcpy.S create mode 100644 arch/sparc/lib/NG2page.S create mode 100644 arch/sparc/lib/NG2patch.S create mode 100644 arch/sparc/lib/NGbzero.S create mode 100644 arch/sparc/lib/NGcopy_from_user.S create mode 100644 arch/sparc/lib/NGcopy_to_user.S create mode 100644 arch/sparc/lib/NGmemcpy.S create mode 100644 arch/sparc/lib/NGpage.S create mode 100644 arch/sparc/lib/NGpatch.S create mode 100644 arch/sparc/lib/PeeCeeI.c create mode 100644 arch/sparc/lib/U1copy_from_user.S create mode 100644 arch/sparc/lib/U1copy_to_user.S create mode 100644 arch/sparc/lib/U1memcpy.S create mode 100644 arch/sparc/lib/U3copy_from_user.S create mode 100644 arch/sparc/lib/U3copy_to_user.S create mode 100644 arch/sparc/lib/U3memcpy.S create mode 100644 arch/sparc/lib/U3patch.S create mode 100644 arch/sparc/lib/VISsave.S create mode 100644 arch/sparc/lib/atomic_64.S create mode 100644 arch/sparc/lib/bitops.S create mode 100644 arch/sparc/lib/bzero.S create mode 100644 arch/sparc/lib/checksum_64.S create mode 100644 arch/sparc/lib/clear_page.S create mode 100644 arch/sparc/lib/copy_in_user.S create mode 100644 arch/sparc/lib/copy_page.S create mode 100644 arch/sparc/lib/csum_copy.S create mode 100644 arch/sparc/lib/csum_copy_from_user.S create mode 100644 arch/sparc/lib/csum_copy_to_user.S create mode 100644 arch/sparc/lib/ipcsum.S create mode 100644 arch/sparc/lib/mcount.S create mode 100644 arch/sparc/lib/memcmp_64.S create mode 100644 arch/sparc/lib/memmove.S create mode 100644 arch/sparc/lib/memscan_64.S create mode 100644 arch/sparc/lib/rwsem_64.S create mode 100644 arch/sparc/lib/strlen_64.S create mode 100644 arch/sparc/lib/strlen_user_64.S create mode 100644 arch/sparc/lib/strncmp_64.S create mode 100644 arch/sparc/lib/strncpy_from_user_64.S create mode 100644 arch/sparc/lib/user_fixup.c create mode 100644 arch/sparc/lib/xor.S (limited to 'arch/sparc/lib') diff --git a/arch/sparc/lib/GENbzero.S b/arch/sparc/lib/GENbzero.S new file mode 100644 index 000000000000..6a4f956a2f7a --- /dev/null +++ b/arch/sparc/lib/GENbzero.S @@ -0,0 +1,160 @@ +/* GENbzero.S: Generic sparc64 memset/clear_user. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ +#include + +#define EX_ST(x,y) \ +98: x,y; \ + .section .fixup; \ + .align 4; \ +99: retl; \ + mov %o1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + + .align 32 + .text + + .globl GENmemset + .type GENmemset, #function +GENmemset: /* %o0=buf, %o1=pat, %o2=len */ + and %o1, 0xff, %o3 + mov %o2, %o1 + sllx %o3, 8, %g1 + or %g1, %o3, %o2 + sllx %o2, 16, %g1 + or %g1, %o2, %o2 + sllx %o2, 32, %g1 + ba,pt %xcc, 1f + or %g1, %o2, %o2 + + .globl GENbzero + .type GENbzero, #function +GENbzero: + clr %o2 +1: brz,pn %o1, GENbzero_return + mov %o0, %o3 + + /* %o5: saved %asi, restored at GENbzero_done + * %o4: store %asi to use + */ + rd %asi, %o5 + mov ASI_P, %o4 + wr %o4, 0x0, %asi + +GENbzero_from_clear_user: + cmp %o1, 15 + bl,pn %icc, GENbzero_tiny + andcc %o0, 0x7, %g1 + be,pt %xcc, 2f + mov 8, %g2 + sub %g2, %g1, %g1 + sub %o1, %g1, %o1 +1: EX_ST(stba %o2, [%o0 + 0x00] %asi) + subcc %g1, 1, %g1 + bne,pt %xcc, 1b + add %o0, 1, %o0 +2: cmp %o1, 128 + bl,pn %icc, GENbzero_medium + andcc %o0, (64 - 1), %g1 + be,pt %xcc, GENbzero_pre_loop + mov 64, %g2 + sub %g2, %g1, %g1 + sub %o1, %g1, %o1 +1: EX_ST(stxa %o2, [%o0 + 0x00] %asi) + subcc %g1, 8, %g1 + bne,pt %xcc, 1b + add %o0, 8, %o0 + +GENbzero_pre_loop: + andn %o1, (64 - 1), %g1 + sub %o1, %g1, %o1 +GENbzero_loop: + EX_ST(stxa %o2, [%o0 + 0x00] %asi) + EX_ST(stxa %o2, [%o0 + 0x08] %asi) + EX_ST(stxa %o2, [%o0 + 0x10] %asi) + EX_ST(stxa %o2, [%o0 + 0x18] %asi) + EX_ST(stxa %o2, [%o0 + 0x20] %asi) + EX_ST(stxa %o2, [%o0 + 0x28] %asi) + EX_ST(stxa %o2, [%o0 + 0x30] %asi) + EX_ST(stxa %o2, [%o0 + 0x38] %asi) + subcc %g1, 64, %g1 + bne,pt %xcc, GENbzero_loop + add %o0, 64, %o0 + + membar #Sync + wr %o4, 0x0, %asi + brz,pn %o1, GENbzero_done +GENbzero_medium: + andncc %o1, 0x7, %g1 + be,pn %xcc, 2f + sub %o1, %g1, %o1 +1: EX_ST(stxa %o2, [%o0 + 0x00] %asi) + subcc %g1, 8, %g1 + bne,pt %xcc, 1b + add %o0, 8, %o0 +2: brz,pt %o1, GENbzero_done + nop + +GENbzero_tiny: +1: EX_ST(stba %o2, [%o0 + 0x00] %asi) + subcc %o1, 1, %o1 + bne,pt %icc, 1b + add %o0, 1, %o0 + + /* fallthrough */ + +GENbzero_done: + wr %o5, 0x0, %asi + +GENbzero_return: + retl + mov %o3, %o0 + .size GENbzero, .-GENbzero + .size GENmemset, .-GENmemset + + .globl GENclear_user + .type GENclear_user, #function +GENclear_user: /* %o0=buf, %o1=len */ + rd %asi, %o5 + brz,pn %o1, GENbzero_done + clr %o3 + cmp %o5, ASI_AIUS + bne,pn %icc, GENbzero + clr %o2 + ba,pt %xcc, GENbzero_from_clear_user + mov ASI_AIUS, %o4 + .size GENclear_user, .-GENclear_user + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define GEN_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl generic_patch_bzero + .type generic_patch_bzero,#function +generic_patch_bzero: + GEN_DO_PATCH(memset, GENmemset) + GEN_DO_PATCH(__bzero, GENbzero) + GEN_DO_PATCH(__clear_user, GENclear_user) + retl + nop + .size generic_patch_bzero,.-generic_patch_bzero diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S new file mode 100644 index 000000000000..2b9df99e87f9 --- /dev/null +++ b/arch/sparc/lib/GENcopy_from_user.S @@ -0,0 +1,34 @@ +/* GENcopy_from_user.S: Generic sparc64 copy from userspace. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ + +#define EX_LD(x) \ +98: x; \ + .section .fixup; \ + .align 4; \ +99: retl; \ + mov 1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#define FUNC_NAME GENcopy_from_user +#define LOAD(type,addr,dest) type##a [addr] ASI_AIUS, dest +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, memcpy_user_stub; \ + nop +#endif + +#include "GENmemcpy.S" diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S new file mode 100644 index 000000000000..bb3f7084daf9 --- /dev/null +++ b/arch/sparc/lib/GENcopy_to_user.S @@ -0,0 +1,38 @@ +/* GENcopy_to_user.S: Generic sparc64 copy to userspace. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ + +#define EX_ST(x) \ +98: x; \ + .section .fixup; \ + .align 4; \ +99: retl; \ + mov 1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#define FUNC_NAME GENcopy_to_user +#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, memcpy_user_stub; \ + nop +#endif + +#include "GENmemcpy.S" diff --git a/arch/sparc/lib/GENmemcpy.S b/arch/sparc/lib/GENmemcpy.S new file mode 100644 index 000000000000..89358ee94851 --- /dev/null +++ b/arch/sparc/lib/GENmemcpy.S @@ -0,0 +1,121 @@ +/* GENmemcpy.S: Generic sparc64 memcpy. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ + +#ifdef __KERNEL__ +#define GLOBAL_SPARE %g7 +#else +#define GLOBAL_SPARE %g5 +#endif + +#ifndef EX_LD +#define EX_LD(x) x +#endif + +#ifndef EX_ST +#define EX_ST(x) x +#endif + +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +#endif + +#ifndef LOAD +#define LOAD(type,addr,dest) type [addr], dest +#endif + +#ifndef STORE +#define STORE(type,src,addr) type src, [addr] +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME GENmemcpy +#endif + +#ifndef PREAMBLE +#define PREAMBLE +#endif + +#ifndef XCC +#define XCC xcc +#endif + + .register %g2,#scratch + .register %g3,#scratch + + .text + .align 64 + + .globl FUNC_NAME + .type FUNC_NAME,#function +FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ + srlx %o2, 31, %g2 + cmp %g2, 0 + tne %XCC, 5 + PREAMBLE + mov %o0, GLOBAL_SPARE + + cmp %o2, 0 + be,pn %XCC, 85f + or %o0, %o1, %o3 + cmp %o2, 16 + blu,a,pn %XCC, 80f + or %o3, %o2, %o3 + + xor %o0, %o1, %o4 + andcc %o4, 0x7, %g0 + bne,a,pn %XCC, 90f + sub %o0, %o1, %o3 + + and %o0, 0x7, %o4 + sub %o4, 0x8, %o4 + sub %g0, %o4, %o4 + sub %o2, %o4, %o2 +1: subcc %o4, 1, %o4 + EX_LD(LOAD(ldub, %o1, %g1)) + EX_ST(STORE(stb, %g1, %o0)) + add %o1, 1, %o1 + bne,pt %XCC, 1b + add %o0, 1, %o0 + + andn %o2, 0x7, %g1 + sub %o2, %g1, %o2 +1: subcc %g1, 0x8, %g1 + EX_LD(LOAD(ldx, %o1, %g2)) + EX_ST(STORE(stx, %g2, %o0)) + add %o1, 0x8, %o1 + bne,pt %XCC, 1b + add %o0, 0x8, %o0 + + brz,pt %o2, 85f + sub %o0, %o1, %o3 + ba,a,pt %XCC, 90f + + .align 64 +80: /* 0 < len <= 16 */ + andcc %o3, 0x3, %g0 + bne,pn %XCC, 90f + sub %o0, %o1, %o3 + +1: + subcc %o2, 4, %o2 + EX_LD(LOAD(lduw, %o1, %g1)) + EX_ST(STORE(stw, %g1, %o1 + %o3)) + bgu,pt %XCC, 1b + add %o1, 4, %o1 + +85: retl + mov EX_RETVAL(GLOBAL_SPARE), %o0 + + .align 32 +90: + subcc %o2, 1, %o2 + EX_LD(LOAD(ldub, %o1, %g1)) + EX_ST(STORE(stb, %g1, %o1 + %o3)) + bgu,pt %XCC, 90b + add %o1, 1, %o1 + retl + mov EX_RETVAL(GLOBAL_SPARE), %o0 + + .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/GENpage.S b/arch/sparc/lib/GENpage.S new file mode 100644 index 000000000000..2ef9d05f21bc --- /dev/null +++ b/arch/sparc/lib/GENpage.S @@ -0,0 +1,77 @@ +/* GENpage.S: Generic clear and copy page. + * + * Copyright (C) 2007 (davem@davemloft.net) + */ +#include + + .text + .align 32 + +GENcopy_user_page: + set PAGE_SIZE, %g7 +1: ldx [%o1 + 0x00], %o2 + ldx [%o1 + 0x08], %o3 + ldx [%o1 + 0x10], %o4 + ldx [%o1 + 0x18], %o5 + stx %o2, [%o0 + 0x00] + stx %o3, [%o0 + 0x08] + stx %o4, [%o0 + 0x10] + stx %o5, [%o0 + 0x18] + ldx [%o1 + 0x20], %o2 + ldx [%o1 + 0x28], %o3 + ldx [%o1 + 0x30], %o4 + ldx [%o1 + 0x38], %o5 + stx %o2, [%o0 + 0x20] + stx %o3, [%o0 + 0x28] + stx %o4, [%o0 + 0x30] + stx %o5, [%o0 + 0x38] + subcc %g7, 64, %g7 + add %o1, 64, %o1 + bne,pt %xcc, 1b + add %o0, 64, %o0 + retl + nop + +GENclear_page: +GENclear_user_page: + set PAGE_SIZE, %g7 +1: stx %g0, [%o0 + 0x00] + stx %g0, [%o0 + 0x08] + stx %g0, [%o0 + 0x10] + stx %g0, [%o0 + 0x18] + stx %g0, [%o0 + 0x20] + stx %g0, [%o0 + 0x28] + stx %g0, [%o0 + 0x30] + stx %g0, [%o0 + 0x38] + subcc %g7, 64, %g7 + bne,pt %xcc, 1b + add %o0, 64, %o0 + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define GEN_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl generic_patch_pageops + .type generic_patch_pageops,#function +generic_patch_pageops: + GEN_DO_PATCH(copy_user_page, GENcopy_user_page) + GEN_DO_PATCH(_clear_page, GENclear_page) + GEN_DO_PATCH(clear_user_page, GENclear_user_page) + retl + nop + .size generic_patch_pageops,.-generic_patch_pageops diff --git a/arch/sparc/lib/GENpatch.S b/arch/sparc/lib/GENpatch.S new file mode 100644 index 000000000000..fab9e89f16bd --- /dev/null +++ b/arch/sparc/lib/GENpatch.S @@ -0,0 +1,33 @@ +/* GENpatch.S: Patch Ultra-I routines with generic variant. + * + * Copyright (C) 2007 David S. Miller + */ + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define GEN_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl generic_patch_copyops + .type generic_patch_copyops,#function +generic_patch_copyops: + GEN_DO_PATCH(memcpy, GENmemcpy) + GEN_DO_PATCH(___copy_from_user, GENcopy_from_user) + GEN_DO_PATCH(___copy_to_user, GENcopy_to_user) + retl + nop + .size generic_patch_copyops,.-generic_patch_copyops diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile index f2650545c774..05ae5c945e35 100644 --- a/arch/sparc/lib/Makefile +++ b/arch/sparc/lib/Makefile @@ -18,5 +18,27 @@ lib-$(CONFIG_SPARC32) += lshrdi3.o ashldi3.o lib-y += rwsem_$(BITS).o lib-$(CONFIG_SPARC32) += muldi3.o bitext.o cmpdi2.o +lib-$(CONFIG_SPARC64) += PeeCeeI.o copy_page.o clear_page.o bzero.o +lib-$(CONFIG_SPARC64) += csum_copy.o csum_copy_from_user.o csum_copy_to_user.o +lib-$(CONFIG_SPARC64) += VISsave.o +lib-$(CONFIG_SPARC64) += bitops.o + +lib-$(CONFIG_SPARC64) += U1memcpy.o U1copy_from_user.o U1copy_to_user.o + +lib-$(CONFIG_SPARC64) += U3memcpy.o U3copy_from_user.o U3copy_to_user.o +lib-$(CONFIG_SPARC64) += U3patch.o + +lib-$(CONFIG_SPARC64) += NGmemcpy.o NGcopy_from_user.o NGcopy_to_user.o +lib-$(CONFIG_SPARC64) += NGpatch.o NGpage.o NGbzero.o + +lib-$(CONFIG_SPARC64) += NG2memcpy.o NG2copy_from_user.o NG2copy_to_user.o +lib-$(CONFIG_SPARC64) += NG2patch.o NG2page.o + +lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o +lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o + +lib-$(CONFIG_SPARC64) += copy_in_user.o user_fixup.o memmove.o +lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o + obj-y += iomap.o obj-$(CONFIG_SPARC32) += atomic32.o diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S new file mode 100644 index 000000000000..c77ef5f22102 --- /dev/null +++ b/arch/sparc/lib/NG2copy_from_user.S @@ -0,0 +1,40 @@ +/* NG2copy_from_user.S: Niagara-2 optimized copy from userspace. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ + +#define EX_LD(x) \ +98: x; \ + .section .fixup; \ + .align 4; \ +99: wr %g0, ASI_AIUS, %asi;\ + retl; \ + mov 1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#ifndef ASI_BLK_AIUS_4V +#define ASI_BLK_AIUS_4V 0x17 +#endif + +#define FUNC_NAME NG2copy_from_user +#define LOAD(type,addr,dest) type##a [addr] %asi, dest +#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_AIUS_4V, dest +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, memcpy_user_stub; \ + nop +#endif + +#include "NG2memcpy.S" diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S new file mode 100644 index 000000000000..4bd4093acbbd --- /dev/null +++ b/arch/sparc/lib/NG2copy_to_user.S @@ -0,0 +1,49 @@ +/* NG2copy_to_user.S: Niagara-2 optimized copy to userspace. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ + +#define EX_ST(x) \ +98: x; \ + .section .fixup; \ + .align 4; \ +99: wr %g0, ASI_AIUS, %asi;\ + retl; \ + mov 1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#ifndef ASI_BLK_AIUS_4V +#define ASI_BLK_AIUS_4V 0x17 +#endif + +#ifndef ASI_BLK_INIT_QUAD_LDD_AIUS +#define ASI_BLK_INIT_QUAD_LDD_AIUS 0x23 +#endif + +#define FUNC_NAME NG2copy_to_user +#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_AIUS +#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS_4V +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, memcpy_user_stub; \ + nop +#endif + +#include "NG2memcpy.S" diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S new file mode 100644 index 000000000000..0aed75653b50 --- /dev/null +++ b/arch/sparc/lib/NG2memcpy.S @@ -0,0 +1,520 @@ +/* NG2memcpy.S: Niagara-2 optimized memcpy. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ + +#ifdef __KERNEL__ +#include +#include +#define GLOBAL_SPARE %g7 +#else +#define ASI_PNF 0x82 +#define ASI_BLK_P 0xf0 +#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 +#define FPRS_FEF 0x04 +#ifdef MEMCPY_DEBUG +#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \ + clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0; +#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#else +#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs +#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#endif +#define GLOBAL_SPARE %g5 +#endif + +#ifndef STORE_ASI +#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P +#else +#define STORE_ASI 0x80 /* ASI_P */ +#endif +#endif + +#ifndef EX_LD +#define EX_LD(x) x +#endif + +#ifndef EX_ST +#define EX_ST(x) x +#endif + +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +#endif + +#ifndef LOAD +#define LOAD(type,addr,dest) type [addr], dest +#endif + +#ifndef LOAD_BLK +#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest +#endif + +#ifndef STORE +#ifndef MEMCPY_DEBUG +#define STORE(type,src,addr) type src, [addr] +#else +#define STORE(type,src,addr) type##a src, [addr] 0x80 +#endif +#endif + +#ifndef STORE_BLK +#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P +#endif + +#ifndef STORE_INIT +#define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME NG2memcpy +#endif + +#ifndef PREAMBLE +#define PREAMBLE +#endif + +#ifndef XCC +#define XCC xcc +#endif + +#define FREG_FROB(x0, x1, x2, x3, x4, x5, x6, x7, x8) \ + faligndata %x0, %x1, %f0; \ + faligndata %x1, %x2, %f2; \ + faligndata %x2, %x3, %f4; \ + faligndata %x3, %x4, %f6; \ + faligndata %x4, %x5, %f8; \ + faligndata %x5, %x6, %f10; \ + faligndata %x6, %x7, %f12; \ + faligndata %x7, %x8, %f14; + +#define FREG_MOVE_1(x0) \ + fmovd %x0, %f0; +#define FREG_MOVE_2(x0, x1) \ + fmovd %x0, %f0; \ + fmovd %x1, %f2; +#define FREG_MOVE_3(x0, x1, x2) \ + fmovd %x0, %f0; \ + fmovd %x1, %f2; \ + fmovd %x2, %f4; +#define FREG_MOVE_4(x0, x1, x2, x3) \ + fmovd %x0, %f0; \ + fmovd %x1, %f2; \ + fmovd %x2, %f4; \ + fmovd %x3, %f6; +#define FREG_MOVE_5(x0, x1, x2, x3, x4) \ + fmovd %x0, %f0; \ + fmovd %x1, %f2; \ + fmovd %x2, %f4; \ + fmovd %x3, %f6; \ + fmovd %x4, %f8; +#define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \ + fmovd %x0, %f0; \ + fmovd %x1, %f2; \ + fmovd %x2, %f4; \ + fmovd %x3, %f6; \ + fmovd %x4, %f8; \ + fmovd %x5, %f10; +#define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \ + fmovd %x0, %f0; \ + fmovd %x1, %f2; \ + fmovd %x2, %f4; \ + fmovd %x3, %f6; \ + fmovd %x4, %f8; \ + fmovd %x5, %f10; \ + fmovd %x6, %f12; +#define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \ + fmovd %x0, %f0; \ + fmovd %x1, %f2; \ + fmovd %x2, %f4; \ + fmovd %x3, %f6; \ + fmovd %x4, %f8; \ + fmovd %x5, %f10; \ + fmovd %x6, %f12; \ + fmovd %x7, %f14; +#define FREG_LOAD_1(base, x0) \ + EX_LD(LOAD(ldd, base + 0x00, %x0)) +#define FREG_LOAD_2(base, x0, x1) \ + EX_LD(LOAD(ldd, base + 0x00, %x0)); \ + EX_LD(LOAD(ldd, base + 0x08, %x1)); +#define FREG_LOAD_3(base, x0, x1, x2) \ + EX_LD(LOAD(ldd, base + 0x00, %x0)); \ + EX_LD(LOAD(ldd, base + 0x08, %x1)); \ + EX_LD(LOAD(ldd, base + 0x10, %x2)); +#define FREG_LOAD_4(base, x0, x1, x2, x3) \ + EX_LD(LOAD(ldd, base + 0x00, %x0)); \ + EX_LD(LOAD(ldd, base + 0x08, %x1)); \ + EX_LD(LOAD(ldd, base + 0x10, %x2)); \ + EX_LD(LOAD(ldd, base + 0x18, %x3)); +#define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \ + EX_LD(LOAD(ldd, base + 0x00, %x0)); \ + EX_LD(LOAD(ldd, base + 0x08, %x1)); \ + EX_LD(LOAD(ldd, base + 0x10, %x2)); \ + EX_LD(LOAD(ldd, base + 0x18, %x3)); \ + EX_LD(LOAD(ldd, base + 0x20, %x4)); +#define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \ + EX_LD(LOAD(ldd, base + 0x00, %x0)); \ + EX_LD(LOAD(ldd, base + 0x08, %x1)); \ + EX_LD(LOAD(ldd, base + 0x10, %x2)); \ + EX_LD(LOAD(ldd, base + 0x18, %x3)); \ + EX_LD(LOAD(ldd, base + 0x20, %x4)); \ + EX_LD(LOAD(ldd, base + 0x28, %x5)); +#define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \ + EX_LD(LOAD(ldd, base + 0x00, %x0)); \ + EX_LD(LOAD(ldd, base + 0x08, %x1)); \ + EX_LD(LOAD(ldd, base + 0x10, %x2)); \ + EX_LD(LOAD(ldd, base + 0x18, %x3)); \ + EX_LD(LOAD(ldd, base + 0x20, %x4)); \ + EX_LD(LOAD(ldd, base + 0x28, %x5)); \ + EX_LD(LOAD(ldd, base + 0x30, %x6)); + + .register %g2,#scratch + .register %g3,#scratch + + .text + .align 64 + + .globl FUNC_NAME + .type FUNC_NAME,#function +FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ + srlx %o2, 31, %g2 + cmp %g2, 0 + tne %xcc, 5 + PREAMBLE + mov %o0, GLOBAL_SPARE + cmp %o2, 0 + be,pn %XCC, 85f + or %o0, %o1, %o3 + cmp %o2, 16 + blu,a,pn %XCC, 80f + or %o3, %o2, %o3 + + /* 2 blocks (128 bytes) is the minimum we can do the block + * copy with. We need to ensure that we'll iterate at least + * once in the block copy loop. At worst we'll need to align + * the destination to a 64-byte boundary which can chew up + * to (64 - 1) bytes from the length before we perform the + * block copy loop. + * + * However, the cut-off point, performance wise, is around + * 4 64-byte blocks. + */ + cmp %o2, (4 * 64) + blu,pt %XCC, 75f + andcc %o3, 0x7, %g0 + + /* %o0: dst + * %o1: src + * %o2: len (known to be >= 128) + * + * The block copy loops can use %o4, %g2, %g3 as + * temporaries while copying the data. %o5 must + * be preserved between VISEntryHalf and VISExitHalf + */ + + LOAD(prefetch, %o1 + 0x000, #one_read) + LOAD(prefetch, %o1 + 0x040, #one_read) + LOAD(prefetch, %o1 + 0x080, #one_read) + + /* Align destination on 64-byte boundary. */ + andcc %o0, (64 - 1), %o4 + be,pt %XCC, 2f + sub %o4, 64, %o4 + sub %g0, %o4, %o4 ! bytes to align dst + sub %o2, %o4, %o2 +1: subcc %o4, 1, %o4 + EX_LD(LOAD(ldub, %o1, %g1)) + EX_ST(STORE(stb, %g1, %o0)) + add %o1, 1, %o1 + bne,pt %XCC, 1b + add %o0, 1, %o0 + +2: + /* Clobbers o5/g1/g2/g3/g7/icc/xcc. We must preserve + * o5 from here until we hit VISExitHalf. + */ + VISEntryHalf + + alignaddr %o1, %g0, %g0 + + add %o1, (64 - 1), %o4 + andn %o4, (64 - 1), %o4 + andn %o2, (64 - 1), %g1 + sub %o2, %g1, %o2 + + and %o1, (64 - 1), %g2 + add %o1, %g1, %o1 + sub %o0, %o4, %g3 + brz,pt %g2, 190f + cmp %g2, 32 + blu,a 5f + cmp %g2, 16 + cmp %g2, 48 + blu,a 4f + cmp %g2, 40 + cmp %g2, 56 + blu 170f + nop + ba,a,pt %xcc, 180f + +4: /* 32 <= low bits < 48 */ + blu 150f + nop + ba,a,pt %xcc, 160f +5: /* 0 < low bits < 32 */ + blu,a 6f + cmp %g2, 8 + cmp %g2, 24 + blu 130f + nop + ba,a,pt %xcc, 140f +6: /* 0 < low bits < 16 */ + bgeu 120f + nop + /* fall through for 0 < low bits < 8 */ +110: sub %o4, 64, %g2 + EX_LD(LOAD_BLK(%g2, %f0)) +1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) + EX_LD(LOAD_BLK(%o4, %f16)) + FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16) + EX_ST(STORE_BLK(%f0, %o4 + %g3)) + FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +120: sub %o4, 56, %g2 + FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12) +1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) + EX_LD(LOAD_BLK(%o4, %f16)) + FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18) + EX_ST(STORE_BLK(%f0, %o4 + %g3)) + FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +130: sub %o4, 48, %g2 + FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10) +1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) + EX_LD(LOAD_BLK(%o4, %f16)) + FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20) + EX_ST(STORE_BLK(%f0, %o4 + %g3)) + FREG_MOVE_6(f20, f22, f24, f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +140: sub %o4, 40, %g2 + FREG_LOAD_5(%g2, f0, f2, f4, f6, f8) +1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) + EX_LD(LOAD_BLK(%o4, %f16)) + FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22) + EX_ST(STORE_BLK(%f0, %o4 + %g3)) + FREG_MOVE_5(f22, f24, f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +150: sub %o4, 32, %g2 + FREG_LOAD_4(%g2, f0, f2, f4, f6) +1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) + EX_LD(LOAD_BLK(%o4, %f16)) + FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24) + EX_ST(STORE_BLK(%f0, %o4 + %g3)) + FREG_MOVE_4(f24, f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +160: sub %o4, 24, %g2 + FREG_LOAD_3(%g2, f0, f2, f4) +1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) + EX_LD(LOAD_BLK(%o4, %f16)) + FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26) + EX_ST(STORE_BLK(%f0, %o4 + %g3)) + FREG_MOVE_3(f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +170: sub %o4, 16, %g2 + FREG_LOAD_2(%g2, f0, f2) +1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) + EX_LD(LOAD_BLK(%o4, %f16)) + FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28) + EX_ST(STORE_BLK(%f0, %o4 + %g3)) + FREG_MOVE_2(f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +180: sub %o4, 8, %g2 + FREG_LOAD_1(%g2, f0) +1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) + EX_LD(LOAD_BLK(%o4, %f16)) + FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30) + EX_ST(STORE_BLK(%f0, %o4 + %g3)) + FREG_MOVE_1(f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +190: +1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) + subcc %g1, 64, %g1 + EX_LD(LOAD_BLK(%o4, %f0)) + EX_ST(STORE_BLK(%f0, %o4 + %g3)) + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + +195: + add %o4, %g3, %o0 + membar #Sync + + VISExitHalf + + /* %o2 contains any final bytes still needed to be copied + * over. If anything is left, we copy it one byte at a time. + */ + brz,pt %o2, 85f + sub %o0, %o1, %o3 + ba,a,pt %XCC, 90f + + .align 64 +75: /* 16 < len <= 64 */ + bne,pn %XCC, 75f + sub %o0, %o1, %o3 + +72: + andn %o2, 0xf, %o4 + and %o2, 0xf, %o2 +1: subcc %o4, 0x10, %o4 + EX_LD(LOAD(ldx, %o1, %o5)) + add %o1, 0x08, %o1 + EX_LD(LOAD(ldx, %o1, %g1)) + sub %o1, 0x08, %o1 + EX_ST(STORE(stx, %o5, %o1 + %o3)) + add %o1, 0x8, %o1 + EX_ST(STORE(stx, %g1, %o1 + %o3)) + bgu,pt %XCC, 1b + add %o1, 0x8, %o1 +73: andcc %o2, 0x8, %g0 + be,pt %XCC, 1f + nop + sub %o2, 0x8, %o2 + EX_LD(LOAD(ldx, %o1, %o5)) + EX_ST(STORE(stx, %o5, %o1 + %o3)) + add %o1, 0x8, %o1 +1: andcc %o2, 0x4, %g0 + be,pt %XCC, 1f + nop + sub %o2, 0x4, %o2 + EX_LD(LOAD(lduw, %o1, %o5)) + EX_ST(STORE(stw, %o5, %o1 + %o3)) + add %o1, 0x4, %o1 +1: cmp %o2, 0 + be,pt %XCC, 85f + nop + ba,pt %xcc, 90f + nop + +75: + andcc %o0, 0x7, %g1 + sub %g1, 0x8, %g1 + be,pn %icc, 2f + sub %g0, %g1, %g1 + sub %o2, %g1, %o2 + +1: subcc %g1, 1, %g1 + EX_LD(LOAD(ldub, %o1, %o5)) + EX_ST(STORE(stb, %o5, %o1 + %o3)) + bgu,pt %icc, 1b + add %o1, 1, %o1 + +2: add %o1, %o3, %o0 + andcc %o1, 0x7, %g1 + bne,pt %icc, 8f + sll %g1, 3, %g1 + + cmp %o2, 16 + bgeu,pt %icc, 72b + nop + ba,a,pt %xcc, 73b + +8: mov 64, %o3 + andn %o1, 0x7, %o1 + EX_LD(LOAD(ldx, %o1, %g2)) + sub %o3, %g1, %o3 + andn %o2, 0x7, %o4 + sllx %g2, %g1, %g2 +1: add %o1, 0x8, %o1 + EX_LD(LOAD(ldx, %o1, %g3)) + subcc %o4, 0x8, %o4 + srlx %g3, %o3, %o5 + or %o5, %g2, %o5 + EX_ST(STORE(stx, %o5, %o0)) + add %o0, 0x8, %o0 + bgu,pt %icc, 1b + sllx %g3, %g1, %g2 + + srl %g1, 3, %g1 + andcc %o2, 0x7, %o2 + be,pn %icc, 85f + add %o1, %g1, %o1 + ba,pt %xcc, 90f + sub %o0, %o1, %o3 + + .align 64 +80: /* 0 < len <= 16 */ + andcc %o3, 0x3, %g0 + bne,pn %XCC, 90f + sub %o0, %o1, %o3 + +1: + subcc %o2, 4, %o2 + EX_LD(LOAD(lduw, %o1, %g1)) + EX_ST(STORE(stw, %g1, %o1 + %o3)) + bgu,pt %XCC, 1b + add %o1, 4, %o1 + +85: retl + mov EX_RETVAL(GLOBAL_SPARE), %o0 + + .align 32 +90: + subcc %o2, 1, %o2 + EX_LD(LOAD(ldub, %o1, %g1)) + EX_ST(STORE(stb, %g1, %o1 + %o3)) + bgu,pt %XCC, 90b + add %o1, 1, %o1 + retl + mov EX_RETVAL(GLOBAL_SPARE), %o0 + + .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/NG2page.S b/arch/sparc/lib/NG2page.S new file mode 100644 index 000000000000..73b6b7c72cbf --- /dev/null +++ b/arch/sparc/lib/NG2page.S @@ -0,0 +1,61 @@ +/* NG2page.S: Niagara-2 optimized clear and copy page. + * + * Copyright (C) 2007 (davem@davemloft.net) + */ + +#include +#include +#include + + .text + .align 32 + + /* This is heavily simplified from the sun4u variants + * because Niagara-2 does not have any D-cache aliasing issues. + */ +NG2copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ + prefetch [%o1 + 0x00], #one_read + prefetch [%o1 + 0x40], #one_read + VISEntryHalf + set PAGE_SIZE, %g7 + sub %o0, %o1, %g3 +1: stxa %g0, [%o1 + %g3] ASI_BLK_INIT_QUAD_LDD_P + subcc %g7, 64, %g7 + ldda [%o1] ASI_BLK_P, %f0 + stda %f0, [%o1 + %g3] ASI_BLK_P + add %o1, 64, %o1 + bne,pt %xcc, 1b + prefetch [%o1 + 0x40], #one_read + membar #Sync + VISExitHalf + retl + nop + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define NG_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl niagara2_patch_pageops + .type niagara2_patch_pageops,#function +niagara2_patch_pageops: + NG_DO_PATCH(copy_user_page, NG2copy_user_page) + NG_DO_PATCH(_clear_page, NGclear_page) + NG_DO_PATCH(clear_user_page, NGclear_user_page) + retl + nop + .size niagara2_patch_pageops,.-niagara2_patch_pageops diff --git a/arch/sparc/lib/NG2patch.S b/arch/sparc/lib/NG2patch.S new file mode 100644 index 000000000000..28c36f06a6d1 --- /dev/null +++ b/arch/sparc/lib/NG2patch.S @@ -0,0 +1,33 @@ +/* NG2patch.S: Patch Ultra-I routines with Niagara-2 variant. + * + * Copyright (C) 2007 David S. Miller + */ + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define NG_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl niagara2_patch_copyops + .type niagara2_patch_copyops,#function +niagara2_patch_copyops: + NG_DO_PATCH(memcpy, NG2memcpy) + NG_DO_PATCH(___copy_from_user, NG2copy_from_user) + NG_DO_PATCH(___copy_to_user, NG2copy_to_user) + retl + nop + .size niagara2_patch_copyops,.-niagara2_patch_copyops diff --git a/arch/sparc/lib/NGbzero.S b/arch/sparc/lib/NGbzero.S new file mode 100644 index 000000000000..814d5f7a45e1 --- /dev/null +++ b/arch/sparc/lib/NGbzero.S @@ -0,0 +1,164 @@ +/* NGbzero.S: Niagara optimized memset/clear_user. + * + * Copyright (C) 2006 David S. Miller (davem@davemloft.net) + */ +#include + +#define EX_ST(x,y) \ +98: x,y; \ + .section .fixup; \ + .align 4; \ +99: retl; \ + mov %o1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + + .text + + .globl NGmemset + .type NGmemset, #function +NGmemset: /* %o0=buf, %o1=pat, %o2=len */ + and %o1, 0xff, %o3 + mov %o2, %o1 + sllx %o3, 8, %g1 + or %g1, %o3, %o2 + sllx %o2, 16, %g1 + or %g1, %o2, %o2 + sllx %o2, 32, %g1 + ba,pt %xcc, 1f + or %g1, %o2, %o2 + + .globl NGbzero + .type NGbzero, #function +NGbzero: + clr %o2 +1: brz,pn %o1, NGbzero_return + mov %o0, %o3 + + /* %o5: saved %asi, restored at NGbzero_done + * %g7: store-init %asi to use + * %o4: non-store-init %asi to use + */ + rd %asi, %o5 + mov ASI_BLK_INIT_QUAD_LDD_P, %g7 + mov ASI_P, %o4 + wr %o4, 0x0, %asi + +NGbzero_from_clear_user: + cmp %o1, 15 + bl,pn %icc, NGbzero_tiny + andcc %o0, 0x7, %g1 + be,pt %xcc, 2f + mov 8, %g2 + sub %g2, %g1, %g1 + sub %o1, %g1, %o1 +1: EX_ST(stba %o2, [%o0 + 0x00] %asi) + subcc %g1, 1, %g1 + bne,pt %xcc, 1b + add %o0, 1, %o0 +2: cmp %o1, 128 + bl,pn %icc, NGbzero_medium + andcc %o0, (64 - 1), %g1 + be,pt %xcc, NGbzero_pre_loop + mov 64, %g2 + sub %g2, %g1, %g1 + sub %o1, %g1, %o1 +1: EX_ST(stxa %o2, [%o0 + 0x00] %asi) + subcc %g1, 8, %g1 + bne,pt %xcc, 1b + add %o0, 8, %o0 + +NGbzero_pre_loop: + wr %g7, 0x0, %asi + andn %o1, (64 - 1), %g1 + sub %o1, %g1, %o1 +NGbzero_loop: + EX_ST(stxa %o2, [%o0 + 0x00] %asi) + EX_ST(stxa %o2, [%o0 + 0x08] %asi) + EX_ST(stxa %o2, [%o0 + 0x10] %asi) + EX_ST(stxa %o2, [%o0 + 0x18] %asi) + EX_ST(stxa %o2, [%o0 + 0x20] %asi) + EX_ST(stxa %o2, [%o0 + 0x28] %asi) + EX_ST(stxa %o2, [%o0 + 0x30] %asi) + EX_ST(stxa %o2, [%o0 + 0x38] %asi) + subcc %g1, 64, %g1 + bne,pt %xcc, NGbzero_loop + add %o0, 64, %o0 + + membar #Sync + wr %o4, 0x0, %asi + brz,pn %o1, NGbzero_done +NGbzero_medium: + andncc %o1, 0x7, %g1 + be,pn %xcc, 2f + sub %o1, %g1, %o1 +1: EX_ST(stxa %o2, [%o0 + 0x00] %asi) + subcc %g1, 8, %g1 + bne,pt %xcc, 1b + add %o0, 8, %o0 +2: brz,pt %o1, NGbzero_done + nop + +NGbzero_tiny: +1: EX_ST(stba %o2, [%o0 + 0x00] %asi) + subcc %o1, 1, %o1 + bne,pt %icc, 1b + add %o0, 1, %o0 + + /* fallthrough */ + +NGbzero_done: + wr %o5, 0x0, %asi + +NGbzero_return: + retl + mov %o3, %o0 + .size NGbzero, .-NGbzero + .size NGmemset, .-NGmemset + + .globl NGclear_user + .type NGclear_user, #function +NGclear_user: /* %o0=buf, %o1=len */ + rd %asi, %o5 + brz,pn %o1, NGbzero_done + clr %o3 + cmp %o5, ASI_AIUS + bne,pn %icc, NGbzero + clr %o2 + mov ASI_BLK_INIT_QUAD_LDD_AIUS, %g7 + ba,pt %xcc, NGbzero_from_clear_user + mov ASI_AIUS, %o4 + .size NGclear_user, .-NGclear_user + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define NG_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl niagara_patch_bzero + .type niagara_patch_bzero,#function +niagara_patch_bzero: + NG_DO_PATCH(memset, NGmemset) + NG_DO_PATCH(__bzero, NGbzero) + NG_DO_PATCH(__clear_user, NGclear_user) + NG_DO_PATCH(tsb_init, NGtsb_init) + retl + nop + .size niagara_patch_bzero,.-niagara_patch_bzero diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S new file mode 100644 index 000000000000..e7f433f71b42 --- /dev/null +++ b/arch/sparc/lib/NGcopy_from_user.S @@ -0,0 +1,37 @@ +/* NGcopy_from_user.S: Niagara optimized copy from userspace. + * + * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) + */ + +#define EX_LD(x) \ +98: x; \ + .section .fixup; \ + .align 4; \ +99: wr %g0, ASI_AIUS, %asi;\ + ret; \ + restore %g0, 1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#define FUNC_NAME NGcopy_from_user +#define LOAD(type,addr,dest) type##a [addr] ASI_AIUS, dest +#define LOAD_TWIN(addr_reg,dest0,dest1) \ + ldda [addr_reg] ASI_BLK_INIT_QUAD_LDD_AIUS, dest0 +#define EX_RETVAL(x) %g0 + +#ifdef __KERNEL__ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, memcpy_user_stub; \ + nop +#endif + +#include "NGmemcpy.S" diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S new file mode 100644 index 000000000000..6ea01c5532a0 --- /dev/null +++ b/arch/sparc/lib/NGcopy_to_user.S @@ -0,0 +1,40 @@ +/* NGcopy_to_user.S: Niagara optimized copy to userspace. + * + * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) + */ + +#define EX_ST(x) \ +98: x; \ + .section .fixup; \ + .align 4; \ +99: wr %g0, ASI_AIUS, %asi;\ + ret; \ + restore %g0, 1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#define FUNC_NAME NGcopy_to_user +#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_AIUS +#define EX_RETVAL(x) %g0 + +#ifdef __KERNEL__ + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, memcpy_user_stub; \ + nop +#endif + +#include "NGmemcpy.S" diff --git a/arch/sparc/lib/NGmemcpy.S b/arch/sparc/lib/NGmemcpy.S new file mode 100644 index 000000000000..96a14caf6966 --- /dev/null +++ b/arch/sparc/lib/NGmemcpy.S @@ -0,0 +1,425 @@ +/* NGmemcpy.S: Niagara optimized memcpy. + * + * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) + */ + +#ifdef __KERNEL__ +#include +#include +#define GLOBAL_SPARE %g7 +#define RESTORE_ASI(TMP) \ + ldub [%g6 + TI_CURRENT_DS], TMP; \ + wr TMP, 0x0, %asi; +#else +#define GLOBAL_SPARE %g5 +#define RESTORE_ASI(TMP) \ + wr %g0, ASI_PNF, %asi +#endif + +#ifdef __sparc_v9__ +#define SAVE_AMOUNT 128 +#else +#define SAVE_AMOUNT 64 +#endif + +#ifndef STORE_ASI +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P +#endif + +#ifndef EX_LD +#define EX_LD(x) x +#endif + +#ifndef EX_ST +#define EX_ST(x) x +#endif + +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +#endif + +#ifndef LOAD +#ifndef MEMCPY_DEBUG +#define LOAD(type,addr,dest) type [addr], dest +#else +#define LOAD(type,addr,dest) type##a [addr] 0x80, dest +#endif +#endif + +#ifndef LOAD_TWIN +#define LOAD_TWIN(addr_reg,dest0,dest1) \ + ldda [addr_reg] ASI_BLK_INIT_QUAD_LDD_P, dest0 +#endif + +#ifndef STORE +#define STORE(type,src,addr) type src, [addr] +#endif + +#ifndef STORE_INIT +#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA +#define STORE_INIT(src,addr) stxa src, [addr] %asi +#else +#define STORE_INIT(src,addr) stx src, [addr + 0x00] +#endif +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME NGmemcpy +#endif + +#ifndef PREAMBLE +#define PREAMBLE +#endif + +#ifndef XCC +#define XCC xcc +#endif + + .register %g2,#scratch + .register %g3,#scratch + + .text + .align 64 + + .globl FUNC_NAME + .type FUNC_NAME,#function +FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ + PREAMBLE + save %sp, -SAVE_AMOUNT, %sp + srlx %i2, 31, %g2 + cmp %g2, 0 + tne %xcc, 5 + mov %i0, %o0 + cmp %i2, 0 + be,pn %XCC, 85f + or %o0, %i1, %i3 + cmp %i2, 16 + blu,a,pn %XCC, 80f + or %i3, %i2, %i3 + + /* 2 blocks (128 bytes) is the minimum we can do the block + * copy with. We need to ensure that we'll iterate at least + * once in the block copy loop. At worst we'll need to align + * the destination to a 64-byte boundary which can chew up + * to (64 - 1) bytes from the length before we perform the + * block copy loop. + */ + cmp %i2, (2 * 64) + blu,pt %XCC, 70f + andcc %i3, 0x7, %g0 + + /* %o0: dst + * %i1: src + * %i2: len (known to be >= 128) + * + * The block copy loops will use %i4/%i5,%g2/%g3 as + * temporaries while copying the data. + */ + + LOAD(prefetch, %i1, #one_read) + wr %g0, STORE_ASI, %asi + + /* Align destination on 64-byte boundary. */ + andcc %o0, (64 - 1), %i4 + be,pt %XCC, 2f + sub %i4, 64, %i4 + sub %g0, %i4, %i4 ! bytes to align dst + sub %i2, %i4, %i2 +1: subcc %i4, 1, %i4 + EX_LD(LOAD(ldub, %i1, %g1)) + EX_ST(STORE(stb, %g1, %o0)) + add %i1, 1, %i1 + bne,pt %XCC, 1b + add %o0, 1, %o0 + + /* If the source is on a 16-byte boundary we can do + * the direct block copy loop. If it is 8-byte aligned + * we can do the 16-byte loads offset by -8 bytes and the + * init stores offset by one register. + * + * If the source is not even 8-byte aligned, we need to do + * shifting and masking (basically integer faligndata). + * + * The careful bit with init stores is that if we store + * to any part of the cache line we have to store the whole + * cacheline else we can end up with corrupt L2 cache line + * contents. Since the loop works on 64-bytes of 64-byte + * aligned store data at a time, this is easy to ensure. + */ +2: + andcc %i1, (16 - 1), %i4 + andn %i2, (64 - 1), %g1 ! block copy loop iterator + be,pt %XCC, 50f + sub %i2, %g1, %i2 ! final sub-block copy bytes + + cmp %i4, 8 + be,pt %XCC, 10f + sub %i1, %i4, %i1 + + /* Neither 8-byte nor 16-byte aligned, shift and mask. */ + and %i4, 0x7, GLOBAL_SPARE + sll GLOBAL_SPARE, 3, GLOBAL_SPARE + mov 64, %i5 + EX_LD(LOAD_TWIN(%i1, %g2, %g3)) + sub %i5, GLOBAL_SPARE, %i5 + mov 16, %o4 + mov 32, %o5 + mov 48, %o7 + mov 64, %i3 + + bg,pn %XCC, 9f + nop + +#define MIX_THREE_WORDS(WORD1, WORD2, WORD3, PRE_SHIFT, POST_SHIFT, TMP) \ + sllx WORD1, POST_SHIFT, WORD1; \ + srlx WORD2, PRE_SHIFT, TMP; \ + sllx WORD2, POST_SHIFT, WORD2; \ + or WORD1, TMP, WORD1; \ + srlx WORD3, PRE_SHIFT, TMP; \ + or WORD2, TMP, WORD2; + +8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3)) + MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1) + LOAD(prefetch, %i1 + %i3, #one_read) + + EX_ST(STORE_INIT(%g2, %o0 + 0x00)) + EX_ST(STORE_INIT(%g3, %o0 + 0x08)) + + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3)) + MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1) + + EX_ST(STORE_INIT(%o2, %o0 + 0x10)) + EX_ST(STORE_INIT(%o3, %o0 + 0x18)) + + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) + MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1) + + EX_ST(STORE_INIT(%g2, %o0 + 0x20)) + EX_ST(STORE_INIT(%g3, %o0 + 0x28)) + + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3)) + add %i1, 64, %i1 + MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1) + + EX_ST(STORE_INIT(%o2, %o0 + 0x30)) + EX_ST(STORE_INIT(%o3, %o0 + 0x38)) + + subcc %g1, 64, %g1 + bne,pt %XCC, 8b + add %o0, 64, %o0 + + ba,pt %XCC, 60f + add %i1, %i4, %i1 + +9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3)) + MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1) + LOAD(prefetch, %i1 + %i3, #one_read) + + EX_ST(STORE_INIT(%g3, %o0 + 0x00)) + EX_ST(STORE_INIT(%o2, %o0 + 0x08)) + + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3)) + MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1) + + EX_ST(STORE_INIT(%o3, %o0 + 0x10)) + EX_ST(STORE_INIT(%g2, %o0 + 0x18)) + + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) + MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1) + + EX_ST(STORE_INIT(%g3, %o0 + 0x20)) + EX_ST(STORE_INIT(%o2, %o0 + 0x28)) + + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3)) + add %i1, 64, %i1 + MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1) + + EX_ST(STORE_INIT(%o3, %o0 + 0x30)) + EX_ST(STORE_INIT(%g2, %o0 + 0x38)) + + subcc %g1, 64, %g1 + bne,pt %XCC, 9b + add %o0, 64, %o0 + + ba,pt %XCC, 60f + add %i1, %i4, %i1 + +10: /* Destination is 64-byte aligned, source was only 8-byte + * aligned but it has been subtracted by 8 and we perform + * one twin load ahead, then add 8 back into source when + * we finish the loop. + */ + EX_LD(LOAD_TWIN(%i1, %o4, %o5)) + mov 16, %o7 + mov 32, %g2 + mov 48, %g3 + mov 64, %o1 +1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) + LOAD(prefetch, %i1 + %o1, #one_read) + EX_ST(STORE_INIT(%o5, %o0 + 0x00)) ! initializes cache line + EX_ST(STORE_INIT(%o2, %o0 + 0x08)) + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5)) + EX_ST(STORE_INIT(%o3, %o0 + 0x10)) + EX_ST(STORE_INIT(%o4, %o0 + 0x18)) + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3)) + EX_ST(STORE_INIT(%o5, %o0 + 0x20)) + EX_ST(STORE_INIT(%o2, %o0 + 0x28)) + EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5)) + add %i1, 64, %i1 + EX_ST(STORE_INIT(%o3, %o0 + 0x30)) + EX_ST(STORE_INIT(%o4, %o0 + 0x38)) + subcc %g1, 64, %g1 + bne,pt %XCC, 1b + add %o0, 64, %o0 + + ba,pt %XCC, 60f + add %i1, 0x8, %i1 + +50: /* Destination is 64-byte aligned, and source is 16-byte + * aligned. + */ + mov 16, %o7 + mov 32, %g2 + mov 48, %g3 + mov 64, %o1 +1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5)) + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) + LOAD(prefetch, %i1 + %o1, #one_read) + EX_ST(STORE_INIT(%o4, %o0 + 0x00)) ! initializes cache line + EX_ST(STORE_INIT(%o5, %o0 + 0x08)) + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5)) + EX_ST(STORE_INIT(%o2, %o0 + 0x10)) + EX_ST(STORE_INIT(%o3, %o0 + 0x18)) + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3)) + add %i1, 64, %i1 + EX_ST(STORE_INIT(%o4, %o0 + 0x20)) + EX_ST(STORE_INIT(%o5, %o0 + 0x28)) + EX_ST(STORE_INIT(%o2, %o0 + 0x30)) + EX_ST(STORE_INIT(%o3, %o0 + 0x38)) + subcc %g1, 64, %g1 + bne,pt %XCC, 1b + add %o0, 64, %o0 + /* fall through */ + +60: + membar #Sync + + /* %i2 contains any final bytes still needed to be copied + * over. If anything is left, we copy it one byte at a time. + */ + RESTORE_ASI(%i3) + brz,pt %i2, 85f + sub %o0, %i1, %i3 + ba,a,pt %XCC, 90f + + .align 64 +70: /* 16 < len <= 64 */ + bne,pn %XCC, 75f + sub %o0, %i1, %i3 + +72: + andn %i2, 0xf, %i4 + and %i2, 0xf, %i2 +1: subcc %i4, 0x10, %i4 + EX_LD(LOAD(ldx, %i1, %o4)) + add %i1, 0x08, %i1 + EX_LD(LOAD(ldx, %i1, %g1)) + sub %i1, 0x08, %i1 + EX_ST(STORE(stx, %o4, %i1 + %i3)) + add %i1, 0x8, %i1 + EX_ST(STORE(stx, %g1, %i1 + %i3)) + bgu,pt %XCC, 1b + add %i1, 0x8, %i1 +73: andcc %i2, 0x8, %g0 + be,pt %XCC, 1f + nop + sub %i2, 0x8, %i2 + EX_LD(LOAD(ldx, %i1, %o4)) + EX_ST(STORE(stx, %o4, %i1 + %i3)) + add %i1, 0x8, %i1 +1: andcc %i2, 0x4, %g0 + be,pt %XCC, 1f + nop + sub %i2, 0x4, %i2 + EX_LD(LOAD(lduw, %i1, %i5)) + EX_ST(STORE(stw, %i5, %i1 + %i3)) + add %i1, 0x4, %i1 +1: cmp %i2, 0 + be,pt %XCC, 85f + nop + ba,pt %xcc, 90f + nop + +75: + andcc %o0, 0x7, %g1 + sub %g1, 0x8, %g1 + be,pn %icc, 2f + sub %g0, %g1, %g1 + sub %i2, %g1, %i2 + +1: subcc %g1, 1, %g1 + EX_LD(LOAD(ldub, %i1, %i5)) + EX_ST(STORE(stb, %i5, %i1 + %i3)) + bgu,pt %icc, 1b + add %i1, 1, %i1 + +2: add %i1, %i3, %o0 + andcc %i1, 0x7, %g1 + bne,pt %icc, 8f + sll %g1, 3, %g1 + + cmp %i2, 16 + bgeu,pt %icc, 72b + nop + ba,a,pt %xcc, 73b + +8: mov 64, %i3 + andn %i1, 0x7, %i1 + EX_LD(LOAD(ldx, %i1, %g2)) + sub %i3, %g1, %i3 + andn %i2, 0x7, %i4 + sllx %g2, %g1, %g2 +1: add %i1, 0x8, %i1 + EX_LD(LOAD(ldx, %i1, %g3)) + subcc %i4, 0x8, %i4 + srlx %g3, %i3, %i5 + or %i5, %g2, %i5 + EX_ST(STORE(stx, %i5, %o0)) + add %o0, 0x8, %o0 + bgu,pt %icc, 1b + sllx %g3, %g1, %g2 + + srl %g1, 3, %g1 + andcc %i2, 0x7, %i2 + be,pn %icc, 85f + add %i1, %g1, %i1 + ba,pt %xcc, 90f + sub %o0, %i1, %i3 + + .align 64 +80: /* 0 < len <= 16 */ + andcc %i3, 0x3, %g0 + bne,pn %XCC, 90f + sub %o0, %i1, %i3 + +1: + subcc %i2, 4, %i2 + EX_LD(LOAD(lduw, %i1, %g1)) + EX_ST(STORE(stw, %g1, %i1 + %i3)) + bgu,pt %XCC, 1b + add %i1, 4, %i1 + +85: ret + restore EX_RETVAL(%i0), %g0, %o0 + + .align 32 +90: + subcc %i2, 1, %i2 + EX_LD(LOAD(ldub, %i1, %g1)) + EX_ST(STORE(stb, %g1, %i1 + %i3)) + bgu,pt %XCC, 90b + add %i1, 1, %i1 + ret + restore EX_RETVAL(%i0), %g0, %o0 + + .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/NGpage.S b/arch/sparc/lib/NGpage.S new file mode 100644 index 000000000000..428920de05ba --- /dev/null +++ b/arch/sparc/lib/NGpage.S @@ -0,0 +1,99 @@ +/* NGpage.S: Niagara optimize clear and copy page. + * + * Copyright (C) 2006 (davem@davemloft.net) + */ + +#include +#include + + .text + .align 32 + + /* This is heavily simplified from the sun4u variants + * because Niagara does not have any D-cache aliasing issues + * and also we don't need to use the FPU in order to implement + * an optimal page copy/clear. + */ + +NGcopy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ + prefetch [%o1 + 0x00], #one_read + mov 8, %g1 + mov 16, %g2 + mov 24, %g3 + set PAGE_SIZE, %g7 + +1: ldda [%o1 + %g0] ASI_BLK_INIT_QUAD_LDD_P, %o2 + ldda [%o1 + %g2] ASI_BLK_INIT_QUAD_LDD_P, %o4 + prefetch [%o1 + 0x40], #one_read + add %o1, 32, %o1 + stxa %o2, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P + stxa %o3, [%o0 + %g1] ASI_BLK_INIT_QUAD_LDD_P + ldda [%o1 + %g0] ASI_BLK_INIT_QUAD_LDD_P, %o2 + stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P + stxa %o5, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P + ldda [%o1 + %g2] ASI_BLK_INIT_QUAD_LDD_P, %o4 + add %o1, 32, %o1 + add %o0, 32, %o0 + stxa %o2, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P + stxa %o3, [%o0 + %g1] ASI_BLK_INIT_QUAD_LDD_P + stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P + stxa %o5, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P + subcc %g7, 64, %g7 + bne,pt %xcc, 1b + add %o0, 32, %o0 + membar #Sync + retl + nop + + .globl NGclear_page, NGclear_user_page +NGclear_page: /* %o0=dest */ +NGclear_user_page: /* %o0=dest, %o1=vaddr */ + mov 8, %g1 + mov 16, %g2 + mov 24, %g3 + set PAGE_SIZE, %g7 + +1: stxa %g0, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P + stxa %g0, [%o0 + %g1] ASI_BLK_INIT_QUAD_LDD_P + stxa %g0, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P + stxa %g0, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P + add %o0, 32, %o0 + stxa %g0, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P + stxa %g0, [%o0 + %g1] ASI_BLK_INIT_QUAD_LDD_P + stxa %g0, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P + stxa %g0, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P + subcc %g7, 64, %g7 + bne,pt %xcc, 1b + add %o0, 32, %o0 + membar #Sync + retl + nop + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define NG_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl niagara_patch_pageops + .type niagara_patch_pageops,#function +niagara_patch_pageops: + NG_DO_PATCH(copy_user_page, NGcopy_user_page) + NG_DO_PATCH(_clear_page, NGclear_page) + NG_DO_PATCH(clear_user_page, NGclear_user_page) + retl + nop + .size niagara_patch_pageops,.-niagara_patch_pageops diff --git a/arch/sparc/lib/NGpatch.S b/arch/sparc/lib/NGpatch.S new file mode 100644 index 000000000000..3b0674fc3366 --- /dev/null +++ b/arch/sparc/lib/NGpatch.S @@ -0,0 +1,33 @@ +/* NGpatch.S: Patch Ultra-I routines with Niagara variant. + * + * Copyright (C) 2006 David S. Miller + */ + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define NG_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl niagara_patch_copyops + .type niagara_patch_copyops,#function +niagara_patch_copyops: + NG_DO_PATCH(memcpy, NGmemcpy) + NG_DO_PATCH(___copy_from_user, NGcopy_from_user) + NG_DO_PATCH(___copy_to_user, NGcopy_to_user) + retl + nop + .size niagara_patch_copyops,.-niagara_patch_copyops diff --git a/arch/sparc/lib/PeeCeeI.c b/arch/sparc/lib/PeeCeeI.c new file mode 100644 index 000000000000..46053e6ddd7b --- /dev/null +++ b/arch/sparc/lib/PeeCeeI.c @@ -0,0 +1,203 @@ +/* + * PeeCeeI.c: The emerging standard... + * + * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include + +void outsb(unsigned long __addr, const void *src, unsigned long count) +{ + void __iomem *addr = (void __iomem *) __addr; + const u8 *p = src; + + while (count--) + outb(*p++, addr); +} + +void outsw(unsigned long __addr, const void *src, unsigned long count) +{ + void __iomem *addr = (void __iomem *) __addr; + + while (count--) { + __raw_writew(*(u16 *)src, addr); + src += sizeof(u16); + } +} + +void outsl(unsigned long __addr, const void *src, unsigned long count) +{ + void __iomem *addr = (void __iomem *) __addr; + u32 l, l2; + + if (!count) + return; + + switch (((unsigned long)src) & 0x3) { + case 0x0: + /* src is naturally aligned */ + while (count--) { + __raw_writel(*(u32 *)src, addr); + src += sizeof(u32); + } + break; + case 0x2: + /* 2-byte alignment */ + while (count--) { + l = (*(u16 *)src) << 16; + l |= *(u16 *)(src + sizeof(u16)); + __raw_writel(l, addr); + src += sizeof(u32); + } + break; + case 0x1: + /* Hold three bytes in l each time, grab a byte from l2 */ + l = (*(u8 *)src) << 24; + l |= (*(u16 *)(src + sizeof(u8))) << 8; + src += sizeof(u8) + sizeof(u16); + while (count--) { + l2 = *(u32 *)src; + l |= (l2 >> 24); + __raw_writel(l, addr); + l = l2 << 8; + src += sizeof(u32); + } + break; + case 0x3: + /* Hold a byte in l each time, grab 3 bytes from l2 */ + l = (*(u8 *)src) << 24; + src += sizeof(u8); + while (count--) { + l2 = *(u32 *)src; + l |= (l2 >> 8); + __raw_writel(l, addr); + l = l2 << 24; + src += sizeof(u32); + } + break; + } +} + +void insb(unsigned long __addr, void *dst, unsigned long count) +{ + void __iomem *addr = (void __iomem *) __addr; + + if (count) { + u32 *pi; + u8 *pb = dst; + + while ((((unsigned long)pb) & 0x3) && count--) + *pb++ = inb(addr); + pi = (u32 *)pb; + while (count >= 4) { + u32 w; + + w = (inb(addr) << 24); + w |= (inb(addr) << 16); + w |= (inb(addr) << 8); + w |= (inb(addr) << 0); + *pi++ = w; + count -= 4; + } + pb = (u8 *)pi; + while (count--) + *pb++ = inb(addr); + } +} + +void insw(unsigned long __addr, void *dst, unsigned long count) +{ + void __iomem *addr = (void __iomem *) __addr; + + if (count) { + u16 *ps = dst; + u32 *pi; + + if (((unsigned long)ps) & 0x2) { + *ps++ = le16_to_cpu(inw(addr)); + count--; + } + pi = (u32 *)ps; + while (count >= 2) { + u32 w; + + w = (le16_to_cpu(inw(addr)) << 16); + w |= (le16_to_cpu(inw(addr)) << 0); + *pi++ = w; + count -= 2; + } + ps = (u16 *)pi; + if (count) + *ps = le16_to_cpu(inw(addr)); + } +} + +void insl(unsigned long __addr, void *dst, unsigned long count) +{ + void __iomem *addr = (void __iomem *) __addr; + + if (count) { + if ((((unsigned long)dst) & 0x3) == 0) { + u32 *pi = dst; + while (count--) + *pi++ = le32_to_cpu(inl(addr)); + } else { + u32 l = 0, l2, *pi; + u16 *ps; + u8 *pb; + + switch (((unsigned long)dst) & 3) { + case 0x2: + ps = dst; + count -= 1; + l = le32_to_cpu(inl(addr)); + *ps++ = l; + pi = (u32 *)ps; + while (count--) { + l2 = le32_to_cpu(inl(addr)); + *pi++ = (l << 16) | (l2 >> 16); + l = l2; + } + ps = (u16 *)pi; + *ps = l; + break; + + case 0x1: + pb = dst; + count -= 1; + l = le32_to_cpu(inl(addr)); + *pb++ = l >> 24; + ps = (u16 *)pb; + *ps++ = ((l >> 8) & 0xffff); + pi = (u32 *)ps; + while (count--) { + l2 = le32_to_cpu(inl(addr)); + *pi++ = (l << 24) | (l2 >> 8); + l = l2; + } + pb = (u8 *)pi; + *pb = l; + break; + + case 0x3: + pb = (u8 *)dst; + count -= 1; + l = le32_to_cpu(inl(addr)); + *pb++ = l >> 24; + pi = (u32 *)pb; + while (count--) { + l2 = le32_to_cpu(inl(addr)); + *pi++ = (l << 8) | (l2 >> 24); + l = l2; + } + ps = (u16 *)pi; + *ps++ = ((l >> 8) & 0xffff); + pb = (u8 *)ps; + *pb = l; + break; + } + } + } +} + diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S new file mode 100644 index 000000000000..3192b0bf4fab --- /dev/null +++ b/arch/sparc/lib/U1copy_from_user.S @@ -0,0 +1,33 @@ +/* U1copy_from_user.S: UltraSparc-I/II/IIi/IIe optimized copy from userspace. + * + * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) + */ + +#define EX_LD(x) \ +98: x; \ + .section .fixup; \ + .align 4; \ +99: retl; \ + mov 1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + +#define FUNC_NAME ___copy_from_user +#define LOAD(type,addr,dest) type##a [addr] %asi, dest +#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_AIUS, dest +#define EX_RETVAL(x) 0 + + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, memcpy_user_stub; \ + nop; \ + +#include "U1memcpy.S" diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S new file mode 100644 index 000000000000..d1210ffb0b82 --- /dev/null +++ b/arch/sparc/lib/U1copy_to_user.S @@ -0,0 +1,33 @@ +/* U1copy_to_user.S: UltraSparc-I/II/IIi/IIe optimized copy to userspace. + * + * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) + */ + +#define EX_ST(x) \ +98: x; \ + .section .fixup; \ + .align 4; \ +99: retl; \ + mov 1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + +#define FUNC_NAME ___copy_to_user +#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS +#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS +#define EX_RETVAL(x) 0 + + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, memcpy_user_stub; \ + nop; \ + +#include "U1memcpy.S" diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S new file mode 100644 index 000000000000..bafd2fc07acb --- /dev/null +++ b/arch/sparc/lib/U1memcpy.S @@ -0,0 +1,563 @@ +/* U1memcpy.S: UltraSPARC-I/II/IIi/IIe optimized memcpy. + * + * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com) + * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz) + */ + +#ifdef __KERNEL__ +#include +#include +#define GLOBAL_SPARE g7 +#else +#define GLOBAL_SPARE g5 +#define ASI_BLK_P 0xf0 +#define FPRS_FEF 0x04 +#ifdef MEMCPY_DEBUG +#define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \ + clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0; +#define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#else +#define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs +#define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#endif +#endif + +#ifndef EX_LD +#define EX_LD(x) x +#endif + +#ifndef EX_ST +#define EX_ST(x) x +#endif + +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +#endif + +#ifndef LOAD +#define LOAD(type,addr,dest) type [addr], dest +#endif + +#ifndef LOAD_BLK +#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest +#endif + +#ifndef STORE +#define STORE(type,src,addr) type src, [addr] +#endif + +#ifndef STORE_BLK +#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME memcpy +#endif + +#ifndef PREAMBLE +#define PREAMBLE +#endif + +#ifndef XCC +#define XCC xcc +#endif + +#define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, f8, f9) \ + faligndata %f1, %f2, %f48; \ + faligndata %f2, %f3, %f50; \ + faligndata %f3, %f4, %f52; \ + faligndata %f4, %f5, %f54; \ + faligndata %f5, %f6, %f56; \ + faligndata %f6, %f7, %f58; \ + faligndata %f7, %f8, %f60; \ + faligndata %f8, %f9, %f62; + +#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \ + EX_LD(LOAD_BLK(%src, %fdest)); \ + EX_ST(STORE_BLK(%fsrc, %dest)); \ + add %src, 0x40, %src; \ + subcc %len, 0x40, %len; \ + be,pn %xcc, jmptgt; \ + add %dest, 0x40, %dest; \ + +#define LOOP_CHUNK1(src, dest, len, branch_dest) \ + MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest) +#define LOOP_CHUNK2(src, dest, len, branch_dest) \ + MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest) +#define LOOP_CHUNK3(src, dest, len, branch_dest) \ + MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest) + +#define DO_SYNC membar #Sync; +#define STORE_SYNC(dest, fsrc) \ + EX_ST(STORE_BLK(%fsrc, %dest)); \ + add %dest, 0x40, %dest; \ + DO_SYNC + +#define STORE_JUMP(dest, fsrc, target) \ + EX_ST(STORE_BLK(%fsrc, %dest)); \ + add %dest, 0x40, %dest; \ + ba,pt %xcc, target; \ + nop; + +#define FINISH_VISCHUNK(dest, f0, f1, left) \ + subcc %left, 8, %left;\ + bl,pn %xcc, 95f; \ + faligndata %f0, %f1, %f48; \ + EX_ST(STORE(std, %f48, %dest)); \ + add %dest, 8, %dest; + +#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ + subcc %left, 8, %left; \ + bl,pn %xcc, 95f; \ + fsrc1 %f0, %f1; + +#define UNEVEN_VISCHUNK(dest, f0, f1, left) \ + UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ + ba,a,pt %xcc, 93f; + + .register %g2,#scratch + .register %g3,#scratch + + .text + .align 64 + + .globl FUNC_NAME + .type FUNC_NAME,#function +FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ + srlx %o2, 31, %g2 + cmp %g2, 0 + tne %xcc, 5 + PREAMBLE + mov %o0, %o4 + cmp %o2, 0 + be,pn %XCC, 85f + or %o0, %o1, %o3 + cmp %o2, 16 + blu,a,pn %XCC, 80f + or %o3, %o2, %o3 + + cmp %o2, (5 * 64) + blu,pt %XCC, 70f + andcc %o3, 0x7, %g0 + + /* Clobbers o5/g1/g2/g3/g7/icc/xcc. */ + VISEntry + + /* Is 'dst' already aligned on an 64-byte boundary? */ + andcc %o0, 0x3f, %g2 + be,pt %XCC, 2f + + /* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number + * of bytes to copy to make 'dst' 64-byte aligned. We pre- + * subtract this from 'len'. + */ + sub %o0, %o1, %GLOBAL_SPARE + sub %g2, 0x40, %g2 + sub %g0, %g2, %g2 + sub %o2, %g2, %o2 + andcc %g2, 0x7, %g1 + be,pt %icc, 2f + and %g2, 0x38, %g2 + +1: subcc %g1, 0x1, %g1 + EX_LD(LOAD(ldub, %o1 + 0x00, %o3)) + EX_ST(STORE(stb, %o3, %o1 + %GLOBAL_SPARE)) + bgu,pt %XCC, 1b + add %o1, 0x1, %o1 + + add %o1, %GLOBAL_SPARE, %o0 + +2: cmp %g2, 0x0 + and %o1, 0x7, %g1 + be,pt %icc, 3f + alignaddr %o1, %g0, %o1 + + EX_LD(LOAD(ldd, %o1, %f4)) +1: EX_LD(LOAD(ldd, %o1 + 0x8, %f6)) + add %o1, 0x8, %o1 + subcc %g2, 0x8, %g2 + faligndata %f4, %f6, %f0 + EX_ST(STORE(std, %f0, %o0)) + be,pn %icc, 3f + add %o0, 0x8, %o0 + + EX_LD(LOAD(ldd, %o1 + 0x8, %f4)) + add %o1, 0x8, %o1 + subcc %g2, 0x8, %g2 + faligndata %f6, %f4, %f0 + EX_ST(STORE(std, %f0, %o0)) + bne,pt %icc, 1b + add %o0, 0x8, %o0 + + /* Destination is 64-byte aligned. */ +3: + membar #LoadStore | #StoreStore | #StoreLoad + + subcc %o2, 0x40, %GLOBAL_SPARE + add %o1, %g1, %g1 + andncc %GLOBAL_SPARE, (0x40 - 1), %GLOBAL_SPARE + srl %g1, 3, %g2 + sub %o2, %GLOBAL_SPARE, %g3 + andn %o1, (0x40 - 1), %o1 + and %g2, 7, %g2 + andncc %g3, 0x7, %g3 + fmovd %f0, %f2 + sub %g3, 0x8, %g3 + sub %o2, %GLOBAL_SPARE, %o2 + + add %g1, %GLOBAL_SPARE, %g1 + subcc %o2, %g3, %o2 + + EX_LD(LOAD_BLK(%o1, %f0)) + add %o1, 0x40, %o1 + add %g1, %g3, %g1 + EX_LD(LOAD_BLK(%o1, %f16)) + add %o1, 0x40, %o1 + sub %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE + EX_LD(LOAD_BLK(%o1, %f32)) + add %o1, 0x40, %o1 + + /* There are 8 instances of the unrolled loop, + * one for each possible alignment of the + * source buffer. Each loop instance is 452 + * bytes. + */ + sll %g2, 3, %o3 + sub %o3, %g2, %o3 + sllx %o3, 4, %o3 + add %o3, %g2, %o3 + sllx %o3, 2, %g2 +1: rd %pc, %o3 + add %o3, %lo(1f - 1b), %o3 + jmpl %o3 + %g2, %g0 + nop + + .align 64 +1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) + LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) + LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) + LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + ba,pt %xcc, 1b+4 + faligndata %f0, %f2, %f48 +1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) + STORE_SYNC(o0, f48) + FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) + STORE_JUMP(o0, f48, 40f) +2: FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) + STORE_SYNC(o0, f48) + FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) + STORE_JUMP(o0, f48, 48f) +3: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) + STORE_SYNC(o0, f48) + FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) + STORE_JUMP(o0, f48, 56f) + +1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) + LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) + LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) + LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + ba,pt %xcc, 1b+4 + faligndata %f2, %f4, %f48 +1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) + STORE_SYNC(o0, f48) + FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) + STORE_JUMP(o0, f48, 41f) +2: FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) + STORE_SYNC(o0, f48) + FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) + STORE_JUMP(o0, f48, 49f) +3: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) + STORE_SYNC(o0, f48) + FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) + STORE_JUMP(o0, f48, 57f) + +1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) + LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) + LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) + LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + ba,pt %xcc, 1b+4 + faligndata %f4, %f6, %f48 +1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) + STORE_SYNC(o0, f48) + FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) + STORE_JUMP(o0, f48, 42f) +2: FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) + STORE_SYNC(o0, f48) + FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) + STORE_JUMP(o0, f48, 50f) +3: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) + STORE_SYNC(o0, f48) + FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) + STORE_JUMP(o0, f48, 58f) + +1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) + LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) + LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) + LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + ba,pt %xcc, 1b+4 + faligndata %f6, %f8, %f48 +1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) + STORE_SYNC(o0, f48) + FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) + STORE_JUMP(o0, f48, 43f) +2: FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) + STORE_SYNC(o0, f48) + FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) + STORE_JUMP(o0, f48, 51f) +3: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) + STORE_SYNC(o0, f48) + FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) + STORE_JUMP(o0, f48, 59f) + +1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) + LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) + LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) + LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + ba,pt %xcc, 1b+4 + faligndata %f8, %f10, %f48 +1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) + STORE_SYNC(o0, f48) + FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) + STORE_JUMP(o0, f48, 44f) +2: FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) + STORE_SYNC(o0, f48) + FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) + STORE_JUMP(o0, f48, 52f) +3: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) + STORE_SYNC(o0, f48) + FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) + STORE_JUMP(o0, f48, 60f) + +1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) + LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) + LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) + LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + ba,pt %xcc, 1b+4 + faligndata %f10, %f12, %f48 +1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) + STORE_SYNC(o0, f48) + FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) + STORE_JUMP(o0, f48, 45f) +2: FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) + STORE_SYNC(o0, f48) + FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) + STORE_JUMP(o0, f48, 53f) +3: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) + STORE_SYNC(o0, f48) + FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) + STORE_JUMP(o0, f48, 61f) + +1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) + LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) + LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) + LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + ba,pt %xcc, 1b+4 + faligndata %f12, %f14, %f48 +1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) + STORE_SYNC(o0, f48) + FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) + STORE_JUMP(o0, f48, 46f) +2: FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) + STORE_SYNC(o0, f48) + FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) + STORE_JUMP(o0, f48, 54f) +3: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) + STORE_SYNC(o0, f48) + FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) + STORE_JUMP(o0, f48, 62f) + +1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) + LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) + LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) + LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + ba,pt %xcc, 1b+4 + faligndata %f14, %f16, %f48 +1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) + STORE_SYNC(o0, f48) + FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) + STORE_JUMP(o0, f48, 47f) +2: FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) + STORE_SYNC(o0, f48) + FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) + STORE_JUMP(o0, f48, 55f) +3: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) + STORE_SYNC(o0, f48) + FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) + STORE_JUMP(o0, f48, 63f) + +40: FINISH_VISCHUNK(o0, f0, f2, g3) +41: FINISH_VISCHUNK(o0, f2, f4, g3) +42: FINISH_VISCHUNK(o0, f4, f6, g3) +43: FINISH_VISCHUNK(o0, f6, f8, g3) +44: FINISH_VISCHUNK(o0, f8, f10, g3) +45: FINISH_VISCHUNK(o0, f10, f12, g3) +46: FINISH_VISCHUNK(o0, f12, f14, g3) +47: UNEVEN_VISCHUNK(o0, f14, f0, g3) +48: FINISH_VISCHUNK(o0, f16, f18, g3) +49: FINISH_VISCHUNK(o0, f18, f20, g3) +50: FINISH_VISCHUNK(o0, f20, f22, g3) +51: FINISH_VISCHUNK(o0, f22, f24, g3) +52: FINISH_VISCHUNK(o0, f24, f26, g3) +53: FINISH_VISCHUNK(o0, f26, f28, g3) +54: FINISH_VISCHUNK(o0, f28, f30, g3) +55: UNEVEN_VISCHUNK(o0, f30, f0, g3) +56: FINISH_VISCHUNK(o0, f32, f34, g3) +57: FINISH_VISCHUNK(o0, f34, f36, g3) +58: FINISH_VISCHUNK(o0, f36, f38, g3) +59: FINISH_VISCHUNK(o0, f38, f40, g3) +60: FINISH_VISCHUNK(o0, f40, f42, g3) +61: FINISH_VISCHUNK(o0, f42, f44, g3) +62: FINISH_VISCHUNK(o0, f44, f46, g3) +63: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3) + +93: EX_LD(LOAD(ldd, %o1, %f2)) + add %o1, 8, %o1 + subcc %g3, 8, %g3 + faligndata %f0, %f2, %f8 + EX_ST(STORE(std, %f8, %o0)) + bl,pn %xcc, 95f + add %o0, 8, %o0 + EX_LD(LOAD(ldd, %o1, %f0)) + add %o1, 8, %o1 + subcc %g3, 8, %g3 + faligndata %f2, %f0, %f8 + EX_ST(STORE(std, %f8, %o0)) + bge,pt %xcc, 93b + add %o0, 8, %o0 + +95: brz,pt %o2, 2f + mov %g1, %o1 + +1: EX_LD(LOAD(ldub, %o1, %o3)) + add %o1, 1, %o1 + subcc %o2, 1, %o2 + EX_ST(STORE(stb, %o3, %o0)) + bne,pt %xcc, 1b + add %o0, 1, %o0 + +2: membar #StoreLoad | #StoreStore + VISExit + retl + mov EX_RETVAL(%o4), %o0 + + .align 64 +70: /* 16 < len <= (5 * 64) */ + bne,pn %XCC, 75f + sub %o0, %o1, %o3 + +72: andn %o2, 0xf, %GLOBAL_SPARE + and %o2, 0xf, %o2 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5)) + EX_LD(LOAD(ldx, %o1 + 0x08, %g1)) + subcc %GLOBAL_SPARE, 0x10, %GLOBAL_SPARE + EX_ST(STORE(stx, %o5, %o1 + %o3)) + add %o1, 0x8, %o1 + EX_ST(STORE(stx, %g1, %o1 + %o3)) + bgu,pt %XCC, 1b + add %o1, 0x8, %o1 +73: andcc %o2, 0x8, %g0 + be,pt %XCC, 1f + nop + EX_LD(LOAD(ldx, %o1, %o5)) + sub %o2, 0x8, %o2 + EX_ST(STORE(stx, %o5, %o1 + %o3)) + add %o1, 0x8, %o1 +1: andcc %o2, 0x4, %g0 + be,pt %XCC, 1f + nop + EX_LD(LOAD(lduw, %o1, %o5)) + sub %o2, 0x4, %o2 + EX_ST(STORE(stw, %o5, %o1 + %o3)) + add %o1, 0x4, %o1 +1: cmp %o2, 0 + be,pt %XCC, 85f + nop + ba,pt %xcc, 90f + nop + +75: andcc %o0, 0x7, %g1 + sub %g1, 0x8, %g1 + be,pn %icc, 2f + sub %g0, %g1, %g1 + sub %o2, %g1, %o2 + +1: EX_LD(LOAD(ldub, %o1, %o5)) + subcc %g1, 1, %g1 + EX_ST(STORE(stb, %o5, %o1 + %o3)) + bgu,pt %icc, 1b + add %o1, 1, %o1 + +2: add %o1, %o3, %o0 + andcc %o1, 0x7, %g1 + bne,pt %icc, 8f + sll %g1, 3, %g1 + + cmp %o2, 16 + bgeu,pt %icc, 72b + nop + ba,a,pt %xcc, 73b + +8: mov 64, %o3 + andn %o1, 0x7, %o1 + EX_LD(LOAD(ldx, %o1, %g2)) + sub %o3, %g1, %o3 + andn %o2, 0x7, %GLOBAL_SPARE + sllx %g2, %g1, %g2 +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3)) + subcc %GLOBAL_SPARE, 0x8, %GLOBAL_SPARE + add %o1, 0x8, %o1 + srlx %g3, %o3, %o5 + or %o5, %g2, %o5 + EX_ST(STORE(stx, %o5, %o0)) + add %o0, 0x8, %o0 + bgu,pt %icc, 1b + sllx %g3, %g1, %g2 + + srl %g1, 3, %g1 + andcc %o2, 0x7, %o2 + be,pn %icc, 85f + add %o1, %g1, %o1 + ba,pt %xcc, 90f + sub %o0, %o1, %o3 + + .align 64 +80: /* 0 < len <= 16 */ + andcc %o3, 0x3, %g0 + bne,pn %XCC, 90f + sub %o0, %o1, %o3 + +1: EX_LD(LOAD(lduw, %o1, %g1)) + subcc %o2, 4, %o2 + EX_ST(STORE(stw, %g1, %o1 + %o3)) + bgu,pt %XCC, 1b + add %o1, 4, %o1 + +85: retl + mov EX_RETVAL(%o4), %o0 + + .align 32 +90: EX_LD(LOAD(ldub, %o1, %g1)) + subcc %o2, 1, %o2 + EX_ST(STORE(stb, %g1, %o1 + %o3)) + bgu,pt %XCC, 90b + add %o1, 1, %o1 + retl + mov EX_RETVAL(%o4), %o0 + + .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/U3copy_from_user.S b/arch/sparc/lib/U3copy_from_user.S new file mode 100644 index 000000000000..f5bfc8d9d216 --- /dev/null +++ b/arch/sparc/lib/U3copy_from_user.S @@ -0,0 +1,22 @@ +/* U3copy_from_user.S: UltraSparc-III optimized copy from userspace. + * + * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) + */ + +#define EX_LD(x) \ +98: x; \ + .section .fixup; \ + .align 4; \ +99: retl; \ + mov 1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + +#define FUNC_NAME U3copy_from_user +#define LOAD(type,addr,dest) type##a [addr] %asi, dest +#define EX_RETVAL(x) 0 + +#include "U3memcpy.S" diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S new file mode 100644 index 000000000000..2334f111bb0c --- /dev/null +++ b/arch/sparc/lib/U3copy_to_user.S @@ -0,0 +1,33 @@ +/* U3copy_to_user.S: UltraSparc-III optimized copy to userspace. + * + * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) + */ + +#define EX_ST(x) \ +98: x; \ + .section .fixup; \ + .align 4; \ +99: retl; \ + mov 1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + +#define FUNC_NAME U3copy_to_user +#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS +#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS +#define EX_RETVAL(x) 0 + + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, memcpy_user_stub; \ + nop; \ + +#include "U3memcpy.S" diff --git a/arch/sparc/lib/U3memcpy.S b/arch/sparc/lib/U3memcpy.S new file mode 100644 index 000000000000..7cae9cc6a204 --- /dev/null +++ b/arch/sparc/lib/U3memcpy.S @@ -0,0 +1,422 @@ +/* U3memcpy.S: UltraSparc-III optimized memcpy. + * + * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) + */ + +#ifdef __KERNEL__ +#include +#include +#define GLOBAL_SPARE %g7 +#else +#define ASI_BLK_P 0xf0 +#define FPRS_FEF 0x04 +#ifdef MEMCPY_DEBUG +#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \ + clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0; +#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#else +#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs +#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#endif +#define GLOBAL_SPARE %g5 +#endif + +#ifndef EX_LD +#define EX_LD(x) x +#endif + +#ifndef EX_ST +#define EX_ST(x) x +#endif + +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +#endif + +#ifndef LOAD +#define LOAD(type,addr,dest) type [addr], dest +#endif + +#ifndef STORE +#define STORE(type,src,addr) type src, [addr] +#endif + +#ifndef STORE_BLK +#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME U3memcpy +#endif + +#ifndef PREAMBLE +#define PREAMBLE +#endif + +#ifndef XCC +#define XCC xcc +#endif + + .register %g2,#scratch + .register %g3,#scratch + + /* Special/non-trivial issues of this code: + * + * 1) %o5 is preserved from VISEntryHalf to VISExitHalf + * 2) Only low 32 FPU registers are used so that only the + * lower half of the FPU register set is dirtied by this + * code. This is especially important in the kernel. + * 3) This code never prefetches cachelines past the end + * of the source buffer. + */ + + .text + .align 64 + + /* The cheetah's flexible spine, oversized liver, enlarged heart, + * slender muscular body, and claws make it the swiftest hunter + * in Africa and the fastest animal on land. Can reach speeds + * of up to 2.4GB per second. + */ + + .globl FUNC_NAME + .type FUNC_NAME,#function +FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ + srlx %o2, 31, %g2 + cmp %g2, 0 + tne %xcc, 5 + PREAMBLE + mov %o0, %o4 + cmp %o2, 0 + be,pn %XCC, 85f + or %o0, %o1, %o3 + cmp %o2, 16 + blu,a,pn %XCC, 80f + or %o3, %o2, %o3 + + cmp %o2, (3 * 64) + blu,pt %XCC, 70f + andcc %o3, 0x7, %g0 + + /* Clobbers o5/g1/g2/g3/g7/icc/xcc. We must preserve + * o5 from here until we hit VISExitHalf. + */ + VISEntryHalf + + /* Is 'dst' already aligned on an 64-byte boundary? */ + andcc %o0, 0x3f, %g2 + be,pt %XCC, 2f + + /* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number + * of bytes to copy to make 'dst' 64-byte aligned. We pre- + * subtract this from 'len'. + */ + sub %o0, %o1, GLOBAL_SPARE + sub %g2, 0x40, %g2 + sub %g0, %g2, %g2 + sub %o2, %g2, %o2 + andcc %g2, 0x7, %g1 + be,pt %icc, 2f + and %g2, 0x38, %g2 + +1: subcc %g1, 0x1, %g1 + EX_LD(LOAD(ldub, %o1 + 0x00, %o3)) + EX_ST(STORE(stb, %o3, %o1 + GLOBAL_SPARE)) + bgu,pt %XCC, 1b + add %o1, 0x1, %o1 + + add %o1, GLOBAL_SPARE, %o0 + +2: cmp %g2, 0x0 + and %o1, 0x7, %g1 + be,pt %icc, 3f + alignaddr %o1, %g0, %o1 + + EX_LD(LOAD(ldd, %o1, %f4)) +1: EX_LD(LOAD(ldd, %o1 + 0x8, %f6)) + add %o1, 0x8, %o1 + subcc %g2, 0x8, %g2 + faligndata %f4, %f6, %f0 + EX_ST(STORE(std, %f0, %o0)) + be,pn %icc, 3f + add %o0, 0x8, %o0 + + EX_LD(LOAD(ldd, %o1 + 0x8, %f4)) + add %o1, 0x8, %o1 + subcc %g2, 0x8, %g2 + faligndata %f6, %f4, %f2 + EX_ST(STORE(std, %f2, %o0)) + bne,pt %icc, 1b + add %o0, 0x8, %o0 + +3: LOAD(prefetch, %o1 + 0x000, #one_read) + LOAD(prefetch, %o1 + 0x040, #one_read) + andn %o2, (0x40 - 1), GLOBAL_SPARE + LOAD(prefetch, %o1 + 0x080, #one_read) + LOAD(prefetch, %o1 + 0x0c0, #one_read) + LOAD(prefetch, %o1 + 0x100, #one_read) + EX_LD(LOAD(ldd, %o1 + 0x000, %f0)) + LOAD(prefetch, %o1 + 0x140, #one_read) + EX_LD(LOAD(ldd, %o1 + 0x008, %f2)) + LOAD(prefetch, %o1 + 0x180, #one_read) + EX_LD(LOAD(ldd, %o1 + 0x010, %f4)) + LOAD(prefetch, %o1 + 0x1c0, #one_read) + faligndata %f0, %f2, %f16 + EX_LD(LOAD(ldd, %o1 + 0x018, %f6)) + faligndata %f2, %f4, %f18 + EX_LD(LOAD(ldd, %o1 + 0x020, %f8)) + faligndata %f4, %f6, %f20 + EX_LD(LOAD(ldd, %o1 + 0x028, %f10)) + faligndata %f6, %f8, %f22 + + EX_LD(LOAD(ldd, %o1 + 0x030, %f12)) + faligndata %f8, %f10, %f24 + EX_LD(LOAD(ldd, %o1 + 0x038, %f14)) + faligndata %f10, %f12, %f26 + EX_LD(LOAD(ldd, %o1 + 0x040, %f0)) + + subcc GLOBAL_SPARE, 0x80, GLOBAL_SPARE + add %o1, 0x40, %o1 + bgu,pt %XCC, 1f + srl GLOBAL_SPARE, 6, %o3 + ba,pt %xcc, 2f + nop + + .align 64 +1: + EX_LD(LOAD(ldd, %o1 + 0x008, %f2)) + faligndata %f12, %f14, %f28 + EX_LD(LOAD(ldd, %o1 + 0x010, %f4)) + faligndata %f14, %f0, %f30 + EX_ST(STORE_BLK(%f16, %o0)) + EX_LD(LOAD(ldd, %o1 + 0x018, %f6)) + faligndata %f0, %f2, %f16 + add %o0, 0x40, %o0 + + EX_LD(LOAD(ldd, %o1 + 0x020, %f8)) + faligndata %f2, %f4, %f18 + EX_LD(LOAD(ldd, %o1 + 0x028, %f10)) + faligndata %f4, %f6, %f20 + EX_LD(LOAD(ldd, %o1 + 0x030, %f12)) + subcc %o3, 0x01, %o3 + faligndata %f6, %f8, %f22 + EX_LD(LOAD(ldd, %o1 + 0x038, %f14)) + + faligndata %f8, %f10, %f24 + EX_LD(LOAD(ldd, %o1 + 0x040, %f0)) + LOAD(prefetch, %o1 + 0x1c0, #one_read) + faligndata %f10, %f12, %f26 + bg,pt %XCC, 1b + add %o1, 0x40, %o1 + + /* Finally we copy the last full 64-byte block. */ +2: + EX_LD(LOAD(ldd, %o1 + 0x008, %f2)) + faligndata %f12, %f14, %f28 + EX_LD(LOAD(ldd, %o1 + 0x010, %f4)) + faligndata %f14, %f0, %f30 + EX_ST(STORE_BLK(%f16, %o0)) + EX_LD(LOAD(ldd, %o1 + 0x018, %f6)) + faligndata %f0, %f2, %f16 + EX_LD(LOAD(ldd, %o1 + 0x020, %f8)) + faligndata %f2, %f4, %f18 + EX_LD(LOAD(ldd, %o1 + 0x028, %f10)) + faligndata %f4, %f6, %f20 + EX_LD(LOAD(ldd, %o1 + 0x030, %f12)) + faligndata %f6, %f8, %f22 + EX_LD(LOAD(ldd, %o1 + 0x038, %f14)) + faligndata %f8, %f10, %f24 + cmp %g1, 0 + be,pt %XCC, 1f + add %o0, 0x40, %o0 + EX_LD(LOAD(ldd, %o1 + 0x040, %f0)) +1: faligndata %f10, %f12, %f26 + faligndata %f12, %f14, %f28 + faligndata %f14, %f0, %f30 + EX_ST(STORE_BLK(%f16, %o0)) + add %o0, 0x40, %o0 + add %o1, 0x40, %o1 + membar #Sync + + /* Now we copy the (len modulo 64) bytes at the end. + * Note how we borrow the %f0 loaded above. + * + * Also notice how this code is careful not to perform a + * load past the end of the src buffer. + */ + and %o2, 0x3f, %o2 + andcc %o2, 0x38, %g2 + be,pn %XCC, 2f + subcc %g2, 0x8, %g2 + be,pn %XCC, 2f + cmp %g1, 0 + + sub %o2, %g2, %o2 + be,a,pt %XCC, 1f + EX_LD(LOAD(ldd, %o1 + 0x00, %f0)) + +1: EX_LD(LOAD(ldd, %o1 + 0x08, %f2)) + add %o1, 0x8, %o1 + subcc %g2, 0x8, %g2 + faligndata %f0, %f2, %f8 + EX_ST(STORE(std, %f8, %o0)) + be,pn %XCC, 2f + add %o0, 0x8, %o0 + EX_LD(LOAD(ldd, %o1 + 0x08, %f0)) + add %o1, 0x8, %o1 + subcc %g2, 0x8, %g2 + faligndata %f2, %f0, %f8 + EX_ST(STORE(std, %f8, %o0)) + bne,pn %XCC, 1b + add %o0, 0x8, %o0 + + /* If anything is left, we copy it one byte at a time. + * Note that %g1 is (src & 0x3) saved above before the + * alignaddr was performed. + */ +2: + cmp %o2, 0 + add %o1, %g1, %o1 + VISExitHalf + be,pn %XCC, 85f + sub %o0, %o1, %o3 + + andcc %g1, 0x7, %g0 + bne,pn %icc, 90f + andcc %o2, 0x8, %g0 + be,pt %icc, 1f + nop + EX_LD(LOAD(ldx, %o1, %o5)) + EX_ST(STORE(stx, %o5, %o1 + %o3)) + add %o1, 0x8, %o1 + +1: andcc %o2, 0x4, %g0 + be,pt %icc, 1f + nop + EX_LD(LOAD(lduw, %o1, %o5)) + EX_ST(STORE(stw, %o5, %o1 + %o3)) + add %o1, 0x4, %o1 + +1: andcc %o2, 0x2, %g0 + be,pt %icc, 1f + nop + EX_LD(LOAD(lduh, %o1, %o5)) + EX_ST(STORE(sth, %o5, %o1 + %o3)) + add %o1, 0x2, %o1 + +1: andcc %o2, 0x1, %g0 + be,pt %icc, 85f + nop + EX_LD(LOAD(ldub, %o1, %o5)) + ba,pt %xcc, 85f + EX_ST(STORE(stb, %o5, %o1 + %o3)) + + .align 64 +70: /* 16 < len <= 64 */ + bne,pn %XCC, 75f + sub %o0, %o1, %o3 + +72: + andn %o2, 0xf, GLOBAL_SPARE + and %o2, 0xf, %o2 +1: subcc GLOBAL_SPARE, 0x10, GLOBAL_SPARE + EX_LD(LOAD(ldx, %o1 + 0x00, %o5)) + EX_LD(LOAD(ldx, %o1 + 0x08, %g1)) + EX_ST(STORE(stx, %o5, %o1 + %o3)) + add %o1, 0x8, %o1 + EX_ST(STORE(stx, %g1, %o1 + %o3)) + bgu,pt %XCC, 1b + add %o1, 0x8, %o1 +73: andcc %o2, 0x8, %g0 + be,pt %XCC, 1f + nop + sub %o2, 0x8, %o2 + EX_LD(LOAD(ldx, %o1, %o5)) + EX_ST(STORE(stx, %o5, %o1 + %o3)) + add %o1, 0x8, %o1 +1: andcc %o2, 0x4, %g0 + be,pt %XCC, 1f + nop + sub %o2, 0x4, %o2 + EX_LD(LOAD(lduw, %o1, %o5)) + EX_ST(STORE(stw, %o5, %o1 + %o3)) + add %o1, 0x4, %o1 +1: cmp %o2, 0 + be,pt %XCC, 85f + nop + ba,pt %xcc, 90f + nop + +75: + andcc %o0, 0x7, %g1 + sub %g1, 0x8, %g1 + be,pn %icc, 2f + sub %g0, %g1, %g1 + sub %o2, %g1, %o2 + +1: subcc %g1, 1, %g1 + EX_LD(LOAD(ldub, %o1, %o5)) + EX_ST(STORE(stb, %o5, %o1 + %o3)) + bgu,pt %icc, 1b + add %o1, 1, %o1 + +2: add %o1, %o3, %o0 + andcc %o1, 0x7, %g1 + bne,pt %icc, 8f + sll %g1, 3, %g1 + + cmp %o2, 16 + bgeu,pt %icc, 72b + nop + ba,a,pt %xcc, 73b + +8: mov 64, %o3 + andn %o1, 0x7, %o1 + EX_LD(LOAD(ldx, %o1, %g2)) + sub %o3, %g1, %o3 + andn %o2, 0x7, GLOBAL_SPARE + sllx %g2, %g1, %g2 +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3)) + subcc GLOBAL_SPARE, 0x8, GLOBAL_SPARE + add %o1, 0x8, %o1 + srlx %g3, %o3, %o5 + or %o5, %g2, %o5 + EX_ST(STORE(stx, %o5, %o0)) + add %o0, 0x8, %o0 + bgu,pt %icc, 1b + sllx %g3, %g1, %g2 + + srl %g1, 3, %g1 + andcc %o2, 0x7, %o2 + be,pn %icc, 85f + add %o1, %g1, %o1 + ba,pt %xcc, 90f + sub %o0, %o1, %o3 + + .align 64 +80: /* 0 < len <= 16 */ + andcc %o3, 0x3, %g0 + bne,pn %XCC, 90f + sub %o0, %o1, %o3 + +1: + subcc %o2, 4, %o2 + EX_LD(LOAD(lduw, %o1, %g1)) + EX_ST(STORE(stw, %g1, %o1 + %o3)) + bgu,pt %XCC, 1b + add %o1, 4, %o1 + +85: retl + mov EX_RETVAL(%o4), %o0 + + .align 32 +90: + subcc %o2, 1, %o2 + EX_LD(LOAD(ldub, %o1, %g1)) + EX_ST(STORE(stb, %g1, %o1 + %o3)) + bgu,pt %XCC, 90b + add %o1, 1, %o1 + retl + mov EX_RETVAL(%o4), %o0 + + .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/U3patch.S b/arch/sparc/lib/U3patch.S new file mode 100644 index 000000000000..ecc302619a6e --- /dev/null +++ b/arch/sparc/lib/U3patch.S @@ -0,0 +1,33 @@ +/* U3patch.S: Patch Ultra-I routines with Ultra-III variant. + * + * Copyright (C) 2004 David S. Miller + */ + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define ULTRA3_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl cheetah_patch_copyops + .type cheetah_patch_copyops,#function +cheetah_patch_copyops: + ULTRA3_DO_PATCH(memcpy, U3memcpy) + ULTRA3_DO_PATCH(___copy_from_user, U3copy_from_user) + ULTRA3_DO_PATCH(___copy_to_user, U3copy_to_user) + retl + nop + .size cheetah_patch_copyops,.-cheetah_patch_copyops diff --git a/arch/sparc/lib/VISsave.S b/arch/sparc/lib/VISsave.S new file mode 100644 index 000000000000..b320ae9e2e2e --- /dev/null +++ b/arch/sparc/lib/VISsave.S @@ -0,0 +1,144 @@ +/* + * VISsave.S: Code for saving FPU register state for + * VIS routines. One should not call this directly, + * but use macros provided in . + * + * Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz) + */ + +#include +#include +#include +#include +#include + + .text + .globl VISenter, VISenterhalf + + /* On entry: %o5=current FPRS value, %g7 is callers address */ + /* May clobber %o5, %g1, %g2, %g3, %g7, %icc, %xcc */ + + /* Nothing special need be done here to handle pre-emption, this + * FPU save/restore mechanism is already preemption safe. + */ + + .align 32 +VISenter: + ldub [%g6 + TI_FPDEPTH], %g1 + brnz,a,pn %g1, 1f + cmp %g1, 1 + stb %g0, [%g6 + TI_FPSAVED] + stx %fsr, [%g6 + TI_XFSR] +9: jmpl %g7 + %g0, %g0 + nop +1: bne,pn %icc, 2f + + srl %g1, 1, %g1 +vis1: ldub [%g6 + TI_FPSAVED], %g3 + stx %fsr, [%g6 + TI_XFSR] + or %g3, %o5, %g3 + stb %g3, [%g6 + TI_FPSAVED] + rd %gsr, %g3 + clr %g1 + ba,pt %xcc, 3f + + stx %g3, [%g6 + TI_GSR] +2: add %g6, %g1, %g3 + cmp %o5, FPRS_DU + be,pn %icc, 6f + sll %g1, 3, %g1 + stb %o5, [%g3 + TI_FPSAVED] + rd %gsr, %g2 + add %g6, %g1, %g3 + stx %g2, [%g3 + TI_GSR] + + add %g6, %g1, %g2 + stx %fsr, [%g2 + TI_XFSR] + sll %g1, 5, %g1 +3: andcc %o5, FPRS_DL|FPRS_DU, %g0 + be,pn %icc, 9b + add %g6, TI_FPREGS, %g2 + andcc %o5, FPRS_DL, %g0 + + be,pn %icc, 4f + add %g6, TI_FPREGS+0x40, %g3 + membar #Sync + stda %f0, [%g2 + %g1] ASI_BLK_P + stda %f16, [%g3 + %g1] ASI_BLK_P + membar #Sync + andcc %o5, FPRS_DU, %g0 + be,pn %icc, 5f +4: add %g1, 128, %g1 + membar #Sync + stda %f32, [%g2 + %g1] ASI_BLK_P + + stda %f48, [%g3 + %g1] ASI_BLK_P +5: membar #Sync + ba,pt %xcc, 80f + nop + + .align 32 +80: jmpl %g7 + %g0, %g0 + nop + +6: ldub [%g3 + TI_FPSAVED], %o5 + or %o5, FPRS_DU, %o5 + add %g6, TI_FPREGS+0x80, %g2 + stb %o5, [%g3 + TI_FPSAVED] + + sll %g1, 5, %g1 + add %g6, TI_FPREGS+0xc0, %g3 + wr %g0, FPRS_FEF, %fprs + membar #Sync + stda %f32, [%g2 + %g1] ASI_BLK_P + stda %f48, [%g3 + %g1] ASI_BLK_P + membar #Sync + ba,pt %xcc, 80f + nop + + .align 32 +80: jmpl %g7 + %g0, %g0 + nop + + .align 32 +VISenterhalf: + ldub [%g6 + TI_FPDEPTH], %g1 + brnz,a,pn %g1, 1f + cmp %g1, 1 + stb %g0, [%g6 + TI_FPSAVED] + stx %fsr, [%g6 + TI_XFSR] + clr %o5 + jmpl %g7 + %g0, %g0 + wr %g0, FPRS_FEF, %fprs + +1: bne,pn %icc, 2f + srl %g1, 1, %g1 + ba,pt %xcc, vis1 + sub %g7, 8, %g7 +2: addcc %g6, %g1, %g3 + sll %g1, 3, %g1 + andn %o5, FPRS_DU, %g2 + stb %g2, [%g3 + TI_FPSAVED] + + rd %gsr, %g2 + add %g6, %g1, %g3 + stx %g2, [%g3 + TI_GSR] + add %g6, %g1, %g2 + stx %fsr, [%g2 + TI_XFSR] + sll %g1, 5, %g1 +3: andcc %o5, FPRS_DL, %g0 + be,pn %icc, 4f + add %g6, TI_FPREGS, %g2 + + add %g6, TI_FPREGS+0x40, %g3 + membar #Sync + stda %f0, [%g2 + %g1] ASI_BLK_P + stda %f16, [%g3 + %g1] ASI_BLK_P + membar #Sync + ba,pt %xcc, 4f + nop + + .align 32 +4: and %o5, FPRS_DU, %o5 + jmpl %g7 + %g0, %g0 + wr %o5, FPRS_FEF, %fprs diff --git a/arch/sparc/lib/atomic_64.S b/arch/sparc/lib/atomic_64.S new file mode 100644 index 000000000000..0268210ca168 --- /dev/null +++ b/arch/sparc/lib/atomic_64.S @@ -0,0 +1,138 @@ +/* atomic.S: These things are too big to do inline. + * + * Copyright (C) 1999, 2007 David S. Miller (davem@davemloft.net) + */ + +#include +#include + + .text + + /* Two versions of the atomic routines, one that + * does not return a value and does not perform + * memory barriers, and a second which returns + * a value and does the barriers. + */ + .globl atomic_add + .type atomic_add,#function +atomic_add: /* %o0 = increment, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) +1: lduw [%o1], %g1 + add %g1, %o0, %g7 + cas [%o1], %g1, %g7 + cmp %g1, %g7 + bne,pn %icc, 2f + nop + retl + nop +2: BACKOFF_SPIN(%o2, %o3, 1b) + .size atomic_add, .-atomic_add + + .globl atomic_sub + .type atomic_sub,#function +atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) +1: lduw [%o1], %g1 + sub %g1, %o0, %g7 + cas [%o1], %g1, %g7 + cmp %g1, %g7 + bne,pn %icc, 2f + nop + retl + nop +2: BACKOFF_SPIN(%o2, %o3, 1b) + .size atomic_sub, .-atomic_sub + + .globl atomic_add_ret + .type atomic_add_ret,#function +atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) +1: lduw [%o1], %g1 + add %g1, %o0, %g7 + cas [%o1], %g1, %g7 + cmp %g1, %g7 + bne,pn %icc, 2f + add %g7, %o0, %g7 + sra %g7, 0, %o0 + retl + nop +2: BACKOFF_SPIN(%o2, %o3, 1b) + .size atomic_add_ret, .-atomic_add_ret + + .globl atomic_sub_ret + .type atomic_sub_ret,#function +atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) +1: lduw [%o1], %g1 + sub %g1, %o0, %g7 + cas [%o1], %g1, %g7 + cmp %g1, %g7 + bne,pn %icc, 2f + sub %g7, %o0, %g7 + sra %g7, 0, %o0 + retl + nop +2: BACKOFF_SPIN(%o2, %o3, 1b) + .size atomic_sub_ret, .-atomic_sub_ret + + .globl atomic64_add + .type atomic64_add,#function +atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) +1: ldx [%o1], %g1 + add %g1, %o0, %g7 + casx [%o1], %g1, %g7 + cmp %g1, %g7 + bne,pn %xcc, 2f + nop + retl + nop +2: BACKOFF_SPIN(%o2, %o3, 1b) + .size atomic64_add, .-atomic64_add + + .globl atomic64_sub + .type atomic64_sub,#function +atomic64_sub: /* %o0 = decrement, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) +1: ldx [%o1], %g1 + sub %g1, %o0, %g7 + casx [%o1], %g1, %g7 + cmp %g1, %g7 + bne,pn %xcc, 2f + nop + retl + nop +2: BACKOFF_SPIN(%o2, %o3, 1b) + .size atomic64_sub, .-atomic64_sub + + .globl atomic64_add_ret + .type atomic64_add_ret,#function +atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) +1: ldx [%o1], %g1 + add %g1, %o0, %g7 + casx [%o1], %g1, %g7 + cmp %g1, %g7 + bne,pn %xcc, 2f + add %g7, %o0, %g7 + mov %g7, %o0 + retl + nop +2: BACKOFF_SPIN(%o2, %o3, 1b) + .size atomic64_add_ret, .-atomic64_add_ret + + .globl atomic64_sub_ret + .type atomic64_sub_ret,#function +atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) +1: ldx [%o1], %g1 + sub %g1, %o0, %g7 + casx [%o1], %g1, %g7 + cmp %g1, %g7 + bne,pn %xcc, 2f + sub %g7, %o0, %g7 + mov %g7, %o0 + retl + nop +2: BACKOFF_SPIN(%o2, %o3, 1b) + .size atomic64_sub_ret, .-atomic64_sub_ret diff --git a/arch/sparc/lib/bitops.S b/arch/sparc/lib/bitops.S new file mode 100644 index 000000000000..2b7228cb8c22 --- /dev/null +++ b/arch/sparc/lib/bitops.S @@ -0,0 +1,141 @@ +/* bitops.S: Sparc64 atomic bit operations. + * + * Copyright (C) 2000, 2007 David S. Miller (davem@davemloft.net) + */ + +#include +#include + + .text + + .globl test_and_set_bit + .type test_and_set_bit,#function +test_and_set_bit: /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) + srlx %o0, 6, %g1 + mov 1, %o2 + sllx %g1, 3, %g3 + and %o0, 63, %g2 + sllx %o2, %g2, %o2 + add %o1, %g3, %o1 +1: ldx [%o1], %g7 + or %g7, %o2, %g1 + casx [%o1], %g7, %g1 + cmp %g7, %g1 + bne,pn %xcc, 2f + and %g7, %o2, %g2 + clr %o0 + movrne %g2, 1, %o0 + retl + nop +2: BACKOFF_SPIN(%o3, %o4, 1b) + .size test_and_set_bit, .-test_and_set_bit + + .globl test_and_clear_bit + .type test_and_clear_bit,#function +test_and_clear_bit: /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) + srlx %o0, 6, %g1 + mov 1, %o2 + sllx %g1, 3, %g3 + and %o0, 63, %g2 + sllx %o2, %g2, %o2 + add %o1, %g3, %o1 +1: ldx [%o1], %g7 + andn %g7, %o2, %g1 + casx [%o1], %g7, %g1 + cmp %g7, %g1 + bne,pn %xcc, 2f + and %g7, %o2, %g2 + clr %o0 + movrne %g2, 1, %o0 + retl + nop +2: BACKOFF_SPIN(%o3, %o4, 1b) + .size test_and_clear_bit, .-test_and_clear_bit + + .globl test_and_change_bit + .type test_and_change_bit,#function +test_and_change_bit: /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) + srlx %o0, 6, %g1 + mov 1, %o2 + sllx %g1, 3, %g3 + and %o0, 63, %g2 + sllx %o2, %g2, %o2 + add %o1, %g3, %o1 +1: ldx [%o1], %g7 + xor %g7, %o2, %g1 + casx [%o1], %g7, %g1 + cmp %g7, %g1 + bne,pn %xcc, 2f + and %g7, %o2, %g2 + clr %o0 + movrne %g2, 1, %o0 + retl + nop +2: BACKOFF_SPIN(%o3, %o4, 1b) + .size test_and_change_bit, .-test_and_change_bit + + .globl set_bit + .type set_bit,#function +set_bit: /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) + srlx %o0, 6, %g1 + mov 1, %o2 + sllx %g1, 3, %g3 + and %o0, 63, %g2 + sllx %o2, %g2, %o2 + add %o1, %g3, %o1 +1: ldx [%o1], %g7 + or %g7, %o2, %g1 + casx [%o1], %g7, %g1 + cmp %g7, %g1 + bne,pn %xcc, 2f + nop + retl + nop +2: BACKOFF_SPIN(%o3, %o4, 1b) + .size set_bit, .-set_bit + + .globl clear_bit + .type clear_bit,#function +clear_bit: /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) + srlx %o0, 6, %g1 + mov 1, %o2 + sllx %g1, 3, %g3 + and %o0, 63, %g2 + sllx %o2, %g2, %o2 + add %o1, %g3, %o1 +1: ldx [%o1], %g7 + andn %g7, %o2, %g1 + casx [%o1], %g7, %g1 + cmp %g7, %g1 + bne,pn %xcc, 2f + nop + retl + nop +2: BACKOFF_SPIN(%o3, %o4, 1b) + .size clear_bit, .-clear_bit + + .globl change_bit + .type change_bit,#function +change_bit: /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) + srlx %o0, 6, %g1 + mov 1, %o2 + sllx %g1, 3, %g3 + and %o0, 63, %g2 + sllx %o2, %g2, %o2 + add %o1, %g3, %o1 +1: ldx [%o1], %g7 + xor %g7, %o2, %g1 + casx [%o1], %g7, %g1 + cmp %g7, %g1 + bne,pn %xcc, 2f + nop + retl + nop +2: BACKOFF_SPIN(%o3, %o4, 1b) + .size change_bit, .-change_bit diff --git a/arch/sparc/lib/bzero.S b/arch/sparc/lib/bzero.S new file mode 100644 index 000000000000..c7bbae8c590f --- /dev/null +++ b/arch/sparc/lib/bzero.S @@ -0,0 +1,158 @@ +/* bzero.S: Simple prefetching memset, bzero, and clear_user + * implementations. + * + * Copyright (C) 2005 David S. Miller + */ + + .text + + .globl __memset + .type __memset, #function +__memset: /* %o0=buf, %o1=pat, %o2=len */ + + .globl memset + .type memset, #function +memset: /* %o0=buf, %o1=pat, %o2=len */ + and %o1, 0xff, %o3 + mov %o2, %o1 + sllx %o3, 8, %g1 + or %g1, %o3, %o2 + sllx %o2, 16, %g1 + or %g1, %o2, %o2 + sllx %o2, 32, %g1 + ba,pt %xcc, 1f + or %g1, %o2, %o2 + + .globl __bzero + .type __bzero, #function +__bzero: /* %o0=buf, %o1=len */ + clr %o2 +1: mov %o0, %o3 + brz,pn %o1, __bzero_done + cmp %o1, 16 + bl,pn %icc, __bzero_tiny + prefetch [%o0 + 0x000], #n_writes + andcc %o0, 0x3, %g0 + be,pt %icc, 2f +1: stb %o2, [%o0 + 0x00] + add %o0, 1, %o0 + andcc %o0, 0x3, %g0 + bne,pn %icc, 1b + sub %o1, 1, %o1 +2: andcc %o0, 0x7, %g0 + be,pt %icc, 3f + stw %o2, [%o0 + 0x00] + sub %o1, 4, %o1 + add %o0, 4, %o0 +3: and %o1, 0x38, %g1 + cmp %o1, 0x40 + andn %o1, 0x3f, %o4 + bl,pn %icc, 5f + and %o1, 0x7, %o1 + prefetch [%o0 + 0x040], #n_writes + prefetch [%o0 + 0x080], #n_writes + prefetch [%o0 + 0x0c0], #n_writes + prefetch [%o0 + 0x100], #n_writes + prefetch [%o0 + 0x140], #n_writes +4: prefetch [%o0 + 0x180], #n_writes + stx %o2, [%o0 + 0x00] + stx %o2, [%o0 + 0x08] + stx %o2, [%o0 + 0x10] + stx %o2, [%o0 + 0x18] + stx %o2, [%o0 + 0x20] + stx %o2, [%o0 + 0x28] + stx %o2, [%o0 + 0x30] + stx %o2, [%o0 + 0x38] + subcc %o4, 0x40, %o4 + bne,pt %icc, 4b + add %o0, 0x40, %o0 + brz,pn %g1, 6f + nop +5: stx %o2, [%o0 + 0x00] + subcc %g1, 8, %g1 + bne,pt %icc, 5b + add %o0, 0x8, %o0 +6: brz,pt %o1, __bzero_done + nop +__bzero_tiny: +1: stb %o2, [%o0 + 0x00] + subcc %o1, 1, %o1 + bne,pt %icc, 1b + add %o0, 1, %o0 +__bzero_done: + retl + mov %o3, %o0 + .size __bzero, .-__bzero + .size __memset, .-__memset + .size memset, .-memset + +#define EX_ST(x,y) \ +98: x,y; \ + .section .fixup; \ + .align 4; \ +99: retl; \ + mov %o1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + + .globl __clear_user + .type __clear_user, #function +__clear_user: /* %o0=buf, %o1=len */ + brz,pn %o1, __clear_user_done + cmp %o1, 16 + bl,pn %icc, __clear_user_tiny + EX_ST(prefetcha [%o0 + 0x00] %asi, #n_writes) + andcc %o0, 0x3, %g0 + be,pt %icc, 2f +1: EX_ST(stba %g0, [%o0 + 0x00] %asi) + add %o0, 1, %o0 + andcc %o0, 0x3, %g0 + bne,pn %icc, 1b + sub %o1, 1, %o1 +2: andcc %o0, 0x7, %g0 + be,pt %icc, 3f + EX_ST(stwa %g0, [%o0 + 0x00] %asi) + sub %o1, 4, %o1 + add %o0, 4, %o0 +3: and %o1, 0x38, %g1 + cmp %o1, 0x40 + andn %o1, 0x3f, %o4 + bl,pn %icc, 5f + and %o1, 0x7, %o1 + EX_ST(prefetcha [%o0 + 0x040] %asi, #n_writes) + EX_ST(prefetcha [%o0 + 0x080] %asi, #n_writes) + EX_ST(prefetcha [%o0 + 0x0c0] %asi, #n_writes) + EX_ST(prefetcha [%o0 + 0x100] %asi, #n_writes) + EX_ST(prefetcha [%o0 + 0x140] %asi, #n_writes) +4: EX_ST(prefetcha [%o0 + 0x180] %asi, #n_writes) + EX_ST(stxa %g0, [%o0 + 0x00] %asi) + EX_ST(stxa %g0, [%o0 + 0x08] %asi) + EX_ST(stxa %g0, [%o0 + 0x10] %asi) + EX_ST(stxa %g0, [%o0 + 0x18] %asi) + EX_ST(stxa %g0, [%o0 + 0x20] %asi) + EX_ST(stxa %g0, [%o0 + 0x28] %asi) + EX_ST(stxa %g0, [%o0 + 0x30] %asi) + EX_ST(stxa %g0, [%o0 + 0x38] %asi) + subcc %o4, 0x40, %o4 + bne,pt %icc, 4b + add %o0, 0x40, %o0 + brz,pn %g1, 6f + nop +5: EX_ST(stxa %g0, [%o0 + 0x00] %asi) + subcc %g1, 8, %g1 + bne,pt %icc, 5b + add %o0, 0x8, %o0 +6: brz,pt %o1, __clear_user_done + nop +__clear_user_tiny: +1: EX_ST(stba %g0, [%o0 + 0x00] %asi) + subcc %o1, 1, %o1 + bne,pt %icc, 1b + add %o0, 1, %o0 +__clear_user_done: + retl + clr %o0 + .size __clear_user, .-__clear_user diff --git a/arch/sparc/lib/checksum_64.S b/arch/sparc/lib/checksum_64.S new file mode 100644 index 000000000000..1d230f693dc4 --- /dev/null +++ b/arch/sparc/lib/checksum_64.S @@ -0,0 +1,173 @@ +/* checksum.S: Sparc V9 optimized checksum code. + * + * Copyright(C) 1995 Linus Torvalds + * Copyright(C) 1995 Miguel de Icaza + * Copyright(C) 1996, 2000 David S. Miller + * Copyright(C) 1997 Jakub Jelinek + * + * derived from: + * Linux/Alpha checksum c-code + * Linux/ix86 inline checksum assembly + * RFC1071 Computing the Internet Checksum (esp. Jacobsons m68k code) + * David Mosberger-Tang for optimized reference c-code + * BSD4.4 portable checksum routine + */ + + .text + +csum_partial_fix_alignment: + /* We checked for zero length already, so there must be + * at least one byte. + */ + be,pt %icc, 1f + nop + ldub [%o0 + 0x00], %o4 + add %o0, 1, %o0 + sub %o1, 1, %o1 +1: andcc %o0, 0x2, %g0 + be,pn %icc, csum_partial_post_align + cmp %o1, 2 + blu,pn %icc, csum_partial_end_cruft + nop + lduh [%o0 + 0x00], %o5 + add %o0, 2, %o0 + sub %o1, 2, %o1 + ba,pt %xcc, csum_partial_post_align + add %o5, %o4, %o4 + + .align 32 + .globl csum_partial +csum_partial: /* %o0=buff, %o1=len, %o2=sum */ + prefetch [%o0 + 0x000], #n_reads + clr %o4 + prefetch [%o0 + 0x040], #n_reads + brz,pn %o1, csum_partial_finish + andcc %o0, 0x3, %g0 + + /* We "remember" whether the lowest bit in the address + * was set in %g7. Because if it is, we have to swap + * upper and lower 8 bit fields of the sum we calculate. + */ + bne,pn %icc, csum_partial_fix_alignment + andcc %o0, 0x1, %g7 + +csum_partial_post_align: + prefetch [%o0 + 0x080], #n_reads + andncc %o1, 0x3f, %o3 + + prefetch [%o0 + 0x0c0], #n_reads + sub %o1, %o3, %o1 + brz,pn %o3, 2f + prefetch [%o0 + 0x100], #n_reads + + /* So that we don't need to use the non-pairing + * add-with-carry instructions we accumulate 32-bit + * values into a 64-bit register. At the end of the + * loop we fold it down to 32-bits and so on. + */ + prefetch [%o0 + 0x140], #n_reads +1: lduw [%o0 + 0x00], %o5 + lduw [%o0 + 0x04], %g1 + lduw [%o0 + 0x08], %g2 + add %o4, %o5, %o4 + lduw [%o0 + 0x0c], %g3 + add %o4, %g1, %o4 + lduw [%o0 + 0x10], %o5 + add %o4, %g2, %o4 + lduw [%o0 + 0x14], %g1 + add %o4, %g3, %o4 + lduw [%o0 + 0x18], %g2 + add %o4, %o5, %o4 + lduw [%o0 + 0x1c], %g3 + add %o4, %g1, %o4 + lduw [%o0 + 0x20], %o5 + add %o4, %g2, %o4 + lduw [%o0 + 0x24], %g1 + add %o4, %g3, %o4 + lduw [%o0 + 0x28], %g2 + add %o4, %o5, %o4 + lduw [%o0 + 0x2c], %g3 + add %o4, %g1, %o4 + lduw [%o0 + 0x30], %o5 + add %o4, %g2, %o4 + lduw [%o0 + 0x34], %g1 + add %o4, %g3, %o4 + lduw [%o0 + 0x38], %g2 + add %o4, %o5, %o4 + lduw [%o0 + 0x3c], %g3 + add %o4, %g1, %o4 + prefetch [%o0 + 0x180], #n_reads + add %o4, %g2, %o4 + subcc %o3, 0x40, %o3 + add %o0, 0x40, %o0 + bne,pt %icc, 1b + add %o4, %g3, %o4 + +2: and %o1, 0x3c, %o3 + brz,pn %o3, 2f + sub %o1, %o3, %o1 +1: lduw [%o0 + 0x00], %o5 + subcc %o3, 0x4, %o3 + add %o0, 0x4, %o0 + bne,pt %icc, 1b + add %o4, %o5, %o4 + +2: + /* fold 64-->32 */ + srlx %o4, 32, %o5 + srl %o4, 0, %o4 + add %o4, %o5, %o4 + srlx %o4, 32, %o5 + srl %o4, 0, %o4 + add %o4, %o5, %o4 + + /* fold 32-->16 */ + sethi %hi(0xffff0000), %g1 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + +csum_partial_end_cruft: + /* %o4 has the 16-bit sum we have calculated so-far. */ + cmp %o1, 2 + blu,pt %icc, 1f + nop + lduh [%o0 + 0x00], %o5 + sub %o1, 2, %o1 + add %o0, 2, %o0 + add %o4, %o5, %o4 +1: brz,pt %o1, 1f + nop + ldub [%o0 + 0x00], %o5 + sub %o1, 1, %o1 + add %o0, 1, %o0 + sllx %o5, 8, %o5 + add %o4, %o5, %o4 +1: + /* fold 32-->16 */ + sethi %hi(0xffff0000), %g1 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + +1: brz,pt %g7, 1f + nop + + /* We started with an odd byte, byte-swap the result. */ + srl %o4, 8, %o5 + and %o4, 0xff, %g1 + sll %g1, 8, %g1 + or %o5, %g1, %o4 + +1: addcc %o2, %o4, %o2 + addc %g0, %o2, %o2 + +csum_partial_finish: + retl + srl %o2, 0, %o0 diff --git a/arch/sparc/lib/clear_page.S b/arch/sparc/lib/clear_page.S new file mode 100644 index 000000000000..77e531f6c2a7 --- /dev/null +++ b/arch/sparc/lib/clear_page.S @@ -0,0 +1,103 @@ +/* clear_page.S: UltraSparc optimized clear page. + * + * Copyright (C) 1996, 1998, 1999, 2000, 2004 David S. Miller (davem@redhat.com) + * Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com) + */ + +#include +#include +#include +#include +#include +#include + + /* What we used to do was lock a TLB entry into a specific + * TLB slot, clear the page with interrupts disabled, then + * restore the original TLB entry. This was great for + * disturbing the TLB as little as possible, but it meant + * we had to keep interrupts disabled for a long time. + * + * Now, we simply use the normal TLB loading mechanism, + * and this makes the cpu choose a slot all by itself. + * Then we do a normal TLB flush on exit. We need only + * disable preemption during the clear. + */ + + .text + + .globl _clear_page +_clear_page: /* %o0=dest */ + ba,pt %xcc, clear_page_common + clr %o4 + + /* This thing is pretty important, it shows up + * on the profiles via do_anonymous_page(). + */ + .align 32 + .globl clear_user_page +clear_user_page: /* %o0=dest, %o1=vaddr */ + lduw [%g6 + TI_PRE_COUNT], %o2 + sethi %uhi(PAGE_OFFSET), %g2 + sethi %hi(PAGE_SIZE), %o4 + + sllx %g2, 32, %g2 + sethi %hi(PAGE_KERNEL_LOCKED), %g3 + + ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3 + sub %o0, %g2, %g1 ! paddr + + and %o1, %o4, %o0 ! vaddr D-cache alias bit + + or %g1, %g3, %g1 ! TTE data + sethi %hi(TLBTEMP_BASE), %o3 + + add %o2, 1, %o4 + add %o0, %o3, %o0 ! TTE vaddr + + /* Disable preemption. */ + mov TLB_TAG_ACCESS, %g3 + stw %o4, [%g6 + TI_PRE_COUNT] + + /* Load TLB entry. */ + rdpr %pstate, %o4 + wrpr %o4, PSTATE_IE, %pstate + stxa %o0, [%g3] ASI_DMMU + stxa %g1, [%g0] ASI_DTLB_DATA_IN + sethi %hi(KERNBASE), %g1 + flush %g1 + wrpr %o4, 0x0, %pstate + + mov 1, %o4 + +clear_page_common: + VISEntryHalf + membar #StoreLoad | #StoreStore | #LoadStore + fzero %f0 + sethi %hi(PAGE_SIZE/64), %o1 + mov %o0, %g1 ! remember vaddr for tlbflush + fzero %f2 + or %o1, %lo(PAGE_SIZE/64), %o1 + faddd %f0, %f2, %f4 + fmuld %f0, %f2, %f6 + faddd %f0, %f2, %f8 + fmuld %f0, %f2, %f10 + + faddd %f0, %f2, %f12 + fmuld %f0, %f2, %f14 +1: stda %f0, [%o0 + %g0] ASI_BLK_P + subcc %o1, 1, %o1 + bne,pt %icc, 1b + add %o0, 0x40, %o0 + membar #Sync + VISExitHalf + + brz,pn %o4, out + nop + + stxa %g0, [%g1] ASI_DMMU_DEMAP + membar #Sync + stw %o2, [%g6 + TI_PRE_COUNT] + +out: retl + nop + diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S new file mode 100644 index 000000000000..650af3f21f78 --- /dev/null +++ b/arch/sparc/lib/copy_in_user.S @@ -0,0 +1,119 @@ +/* copy_in_user.S: Copy from userspace to userspace. + * + * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) + */ + +#include + +#define XCC xcc + +#define EX(x,y) \ +98: x,y; \ + .section .fixup; \ + .align 4; \ +99: retl; \ + mov 1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + + .register %g2,#scratch + .register %g3,#scratch + + .text + .align 32 + + /* Don't try to get too fancy here, just nice and + * simple. This is predominantly used for well aligned + * small copies in the compat layer. It is also used + * to copy register windows around during thread cloning. + */ + + .globl ___copy_in_user + .type ___copy_in_user,#function +___copy_in_user: /* %o0=dst, %o1=src, %o2=len */ + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ + rd %asi, %g1 + cmp %g1, ASI_AIUS + bne,pn %icc, memcpy_user_stub + nop + + cmp %o2, 0 + be,pn %XCC, 85f + or %o0, %o1, %o3 + cmp %o2, 16 + bleu,a,pn %XCC, 80f + or %o3, %o2, %o3 + + /* 16 < len <= 64 */ + andcc %o3, 0x7, %g0 + bne,pn %XCC, 90f + sub %o0, %o1, %o3 + + andn %o2, 0x7, %o4 + and %o2, 0x7, %o2 +1: subcc %o4, 0x8, %o4 + EX(ldxa [%o1] %asi, %o5) + EX(stxa %o5, [%o1 + %o3] ASI_AIUS) + bgu,pt %XCC, 1b + add %o1, 0x8, %o1 + andcc %o2, 0x4, %g0 + be,pt %XCC, 1f + nop + sub %o2, 0x4, %o2 + EX(lduwa [%o1] %asi, %o5) + EX(stwa %o5, [%o1 + %o3] ASI_AIUS) + add %o1, 0x4, %o1 +1: cmp %o2, 0 + be,pt %XCC, 85f + nop + ba,pt %xcc, 90f + nop + +80: /* 0 < len <= 16 */ + andcc %o3, 0x3, %g0 + bne,pn %XCC, 90f + sub %o0, %o1, %o3 + +82: + subcc %o2, 4, %o2 + EX(lduwa [%o1] %asi, %g1) + EX(stwa %g1, [%o1 + %o3] ASI_AIUS) + bgu,pt %XCC, 82b + add %o1, 4, %o1 + +85: retl + clr %o0 + + .align 32 +90: + subcc %o2, 1, %o2 + EX(lduba [%o1] %asi, %g1) + EX(stba %g1, [%o1 + %o3] ASI_AIUS) + bgu,pt %XCC, 90b + add %o1, 1, %o1 + retl + clr %o0 + + .size ___copy_in_user, .-___copy_in_user + + /* Act like copy_{to,in}_user(), ie. return zero instead + * of original destination pointer. This is invoked when + * copy_{to,in}_user() finds that %asi is kernel space. + */ + .globl memcpy_user_stub + .type memcpy_user_stub,#function +memcpy_user_stub: + save %sp, -192, %sp + mov %i0, %o0 + mov %i1, %o1 + call memcpy + mov %i2, %o2 + ret + restore %g0, %g0, %o0 + .size memcpy_user_stub, .-memcpy_user_stub diff --git a/arch/sparc/lib/copy_page.S b/arch/sparc/lib/copy_page.S new file mode 100644 index 000000000000..b243d3b606ba --- /dev/null +++ b/arch/sparc/lib/copy_page.S @@ -0,0 +1,250 @@ +/* clear_page.S: UltraSparc optimized copy page. + * + * Copyright (C) 1996, 1998, 1999, 2000, 2004 David S. Miller (davem@redhat.com) + * Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com) + */ + +#include +#include +#include +#include +#include +#include + + /* What we used to do was lock a TLB entry into a specific + * TLB slot, clear the page with interrupts disabled, then + * restore the original TLB entry. This was great for + * disturbing the TLB as little as possible, but it meant + * we had to keep interrupts disabled for a long time. + * + * Now, we simply use the normal TLB loading mechanism, + * and this makes the cpu choose a slot all by itself. + * Then we do a normal TLB flush on exit. We need only + * disable preemption during the clear. + */ + +#define DCACHE_SIZE (PAGE_SIZE * 2) + +#if (PAGE_SHIFT == 13) +#define PAGE_SIZE_REM 0x80 +#elif (PAGE_SHIFT == 16) +#define PAGE_SIZE_REM 0x100 +#else +#error Wrong PAGE_SHIFT specified +#endif + +#define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \ + fmovd %reg0, %f48; fmovd %reg1, %f50; \ + fmovd %reg2, %f52; fmovd %reg3, %f54; \ + fmovd %reg4, %f56; fmovd %reg5, %f58; \ + fmovd %reg6, %f60; fmovd %reg7, %f62; + + .text + + .align 32 + .globl copy_user_page + .type copy_user_page,#function +copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ + lduw [%g6 + TI_PRE_COUNT], %o4 + sethi %uhi(PAGE_OFFSET), %g2 + sethi %hi(PAGE_SIZE), %o3 + + sllx %g2, 32, %g2 + sethi %hi(PAGE_KERNEL_LOCKED), %g3 + + ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3 + sub %o0, %g2, %g1 ! dest paddr + + sub %o1, %g2, %g2 ! src paddr + + and %o2, %o3, %o0 ! vaddr D-cache alias bit + or %g1, %g3, %g1 ! dest TTE data + + or %g2, %g3, %g2 ! src TTE data + sethi %hi(TLBTEMP_BASE), %o3 + + sethi %hi(DCACHE_SIZE), %o1 + add %o0, %o3, %o0 ! dest TTE vaddr + + add %o4, 1, %o2 + add %o0, %o1, %o1 ! src TTE vaddr + + /* Disable preemption. */ + mov TLB_TAG_ACCESS, %g3 + stw %o2, [%g6 + TI_PRE_COUNT] + + /* Load TLB entries. */ + rdpr %pstate, %o2 + wrpr %o2, PSTATE_IE, %pstate + stxa %o0, [%g3] ASI_DMMU + stxa %g1, [%g0] ASI_DTLB_DATA_IN + membar #Sync + stxa %o1, [%g3] ASI_DMMU + stxa %g2, [%g0] ASI_DTLB_DATA_IN + membar #Sync + wrpr %o2, 0x0, %pstate + +cheetah_copy_page_insn: + ba,pt %xcc, 9f + nop + +1: + VISEntryHalf + membar #StoreLoad | #StoreStore | #LoadStore + sethi %hi((PAGE_SIZE/64)-2), %o2 + mov %o0, %g1 + prefetch [%o1 + 0x000], #one_read + or %o2, %lo((PAGE_SIZE/64)-2), %o2 + prefetch [%o1 + 0x040], #one_read + prefetch [%o1 + 0x080], #one_read + prefetch [%o1 + 0x0c0], #one_read + ldd [%o1 + 0x000], %f0 + prefetch [%o1 + 0x100], #one_read + ldd [%o1 + 0x008], %f2 + prefetch [%o1 + 0x140], #one_read + ldd [%o1 + 0x010], %f4 + prefetch [%o1 + 0x180], #one_read + fmovd %f0, %f16 + ldd [%o1 + 0x018], %f6 + fmovd %f2, %f18 + ldd [%o1 + 0x020], %f8 + fmovd %f4, %f20 + ldd [%o1 + 0x028], %f10 + fmovd %f6, %f22 + ldd [%o1 + 0x030], %f12 + fmovd %f8, %f24 + ldd [%o1 + 0x038], %f14 + fmovd %f10, %f26 + ldd [%o1 + 0x040], %f0 +1: ldd [%o1 + 0x048], %f2 + fmovd %f12, %f28 + ldd [%o1 + 0x050], %f4 + fmovd %f14, %f30 + stda %f16, [%o0] ASI_BLK_P + ldd [%o1 + 0x058], %f6 + fmovd %f0, %f16 + ldd [%o1 + 0x060], %f8 + fmovd %f2, %f18 + ldd [%o1 + 0x068], %f10 + fmovd %f4, %f20 + ldd [%o1 + 0x070], %f12 + fmovd %f6, %f22 + ldd [%o1 + 0x078], %f14 + fmovd %f8, %f24 + ldd [%o1 + 0x080], %f0 + prefetch [%o1 + 0x180], #one_read + fmovd %f10, %f26 + subcc %o2, 1, %o2 + add %o0, 0x40, %o0 + bne,pt %xcc, 1b + add %o1, 0x40, %o1 + + ldd [%o1 + 0x048], %f2 + fmovd %f12, %f28 + ldd [%o1 + 0x050], %f4 + fmovd %f14, %f30 + stda %f16, [%o0] ASI_BLK_P + ldd [%o1 + 0x058], %f6 + fmovd %f0, %f16 + ldd [%o1 + 0x060], %f8 + fmovd %f2, %f18 + ldd [%o1 + 0x068], %f10 + fmovd %f4, %f20 + ldd [%o1 + 0x070], %f12 + fmovd %f6, %f22 + add %o0, 0x40, %o0 + ldd [%o1 + 0x078], %f14 + fmovd %f8, %f24 + fmovd %f10, %f26 + fmovd %f12, %f28 + fmovd %f14, %f30 + stda %f16, [%o0] ASI_BLK_P + membar #Sync + VISExitHalf + ba,pt %xcc, 5f + nop + +9: + VISEntry + ldub [%g6 + TI_FAULT_CODE], %g3 + mov %o0, %g1 + cmp %g3, 0 + rd %asi, %g3 + be,a,pt %icc, 1f + wr %g0, ASI_BLK_P, %asi + wr %g0, ASI_BLK_COMMIT_P, %asi +1: ldda [%o1] ASI_BLK_P, %f0 + add %o1, 0x40, %o1 + ldda [%o1] ASI_BLK_P, %f16 + add %o1, 0x40, %o1 + sethi %hi(PAGE_SIZE), %o2 +1: TOUCH(f0, f2, f4, f6, f8, f10, f12, f14) + ldda [%o1] ASI_BLK_P, %f32 + stda %f48, [%o0] %asi + add %o1, 0x40, %o1 + sub %o2, 0x40, %o2 + add %o0, 0x40, %o0 + TOUCH(f16, f18, f20, f22, f24, f26, f28, f30) + ldda [%o1] ASI_BLK_P, %f0 + stda %f48, [%o0] %asi + add %o1, 0x40, %o1 + sub %o2, 0x40, %o2 + add %o0, 0x40, %o0 + TOUCH(f32, f34, f36, f38, f40, f42, f44, f46) + ldda [%o1] ASI_BLK_P, %f16 + stda %f48, [%o0] %asi + sub %o2, 0x40, %o2 + add %o1, 0x40, %o1 + cmp %o2, PAGE_SIZE_REM + bne,pt %xcc, 1b + add %o0, 0x40, %o0 +#if (PAGE_SHIFT == 16) + TOUCH(f0, f2, f4, f6, f8, f10, f12, f14) + ldda [%o1] ASI_BLK_P, %f32 + stda %f48, [%o0] %asi + add %o1, 0x40, %o1 + sub %o2, 0x40, %o2 + add %o0, 0x40, %o0 + TOUCH(f16, f18, f20, f22, f24, f26, f28, f30) + ldda [%o1] ASI_BLK_P, %f0 + stda %f48, [%o0] %asi + add %o1, 0x40, %o1 + sub %o2, 0x40, %o2 + add %o0, 0x40, %o0 + membar #Sync + stda %f32, [%o0] %asi + add %o0, 0x40, %o0 + stda %f0, [%o0] %asi +#else + membar #Sync + stda %f0, [%o0] %asi + add %o0, 0x40, %o0 + stda %f16, [%o0] %asi +#endif + membar #Sync + wr %g3, 0x0, %asi + VISExit + +5: + stxa %g0, [%g1] ASI_DMMU_DEMAP + membar #Sync + + sethi %hi(DCACHE_SIZE), %g2 + stxa %g0, [%g1 + %g2] ASI_DMMU_DEMAP + membar #Sync + + retl + stw %o4, [%g6 + TI_PRE_COUNT] + + .size copy_user_page, .-copy_user_page + + .globl cheetah_patch_copy_page +cheetah_patch_copy_page: + sethi %hi(0x01000000), %o1 ! NOP + sethi %hi(cheetah_copy_page_insn), %o0 + or %o0, %lo(cheetah_copy_page_insn), %o0 + stw %o1, [%o0] + membar #StoreStore + flush %o0 + retl + nop diff --git a/arch/sparc/lib/csum_copy.S b/arch/sparc/lib/csum_copy.S new file mode 100644 index 000000000000..e566c770a0f6 --- /dev/null +++ b/arch/sparc/lib/csum_copy.S @@ -0,0 +1,309 @@ +/* csum_copy.S: Checksum+copy code for sparc64 + * + * Copyright (C) 2005 David S. Miller + */ + +#ifdef __KERNEL__ +#define GLOBAL_SPARE %g7 +#else +#define GLOBAL_SPARE %g5 +#endif + +#ifndef EX_LD +#define EX_LD(x) x +#endif + +#ifndef EX_ST +#define EX_ST(x) x +#endif + +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +#endif + +#ifndef LOAD +#define LOAD(type,addr,dest) type [addr], dest +#endif + +#ifndef STORE +#define STORE(type,src,addr) type src, [addr] +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME csum_partial_copy_nocheck +#endif + + .register %g2, #scratch + .register %g3, #scratch + + .text + +90: + /* We checked for zero length already, so there must be + * at least one byte. + */ + be,pt %icc, 1f + nop + EX_LD(LOAD(ldub, %o0 + 0x00, %o4)) + add %o0, 1, %o0 + sub %o2, 1, %o2 + EX_ST(STORE(stb, %o4, %o1 + 0x00)) + add %o1, 1, %o1 +1: andcc %o0, 0x2, %g0 + be,pn %icc, 80f + cmp %o2, 2 + blu,pn %icc, 60f + nop + EX_LD(LOAD(lduh, %o0 + 0x00, %o5)) + add %o0, 2, %o0 + sub %o2, 2, %o2 + EX_ST(STORE(sth, %o5, %o1 + 0x00)) + add %o1, 2, %o1 + ba,pt %xcc, 80f + add %o5, %o4, %o4 + + .globl FUNC_NAME +FUNC_NAME: /* %o0=src, %o1=dst, %o2=len, %o3=sum */ + LOAD(prefetch, %o0 + 0x000, #n_reads) + xor %o0, %o1, %g1 + clr %o4 + andcc %g1, 0x3, %g0 + bne,pn %icc, 95f + LOAD(prefetch, %o0 + 0x040, #n_reads) + + brz,pn %o2, 70f + andcc %o0, 0x3, %g0 + + /* We "remember" whether the lowest bit in the address + * was set in GLOBAL_SPARE. Because if it is, we have to swap + * upper and lower 8 bit fields of the sum we calculate. + */ + bne,pn %icc, 90b + andcc %o0, 0x1, GLOBAL_SPARE + +80: + LOAD(prefetch, %o0 + 0x080, #n_reads) + andncc %o2, 0x3f, %g3 + + LOAD(prefetch, %o0 + 0x0c0, #n_reads) + sub %o2, %g3, %o2 + brz,pn %g3, 2f + LOAD(prefetch, %o0 + 0x100, #n_reads) + + /* So that we don't need to use the non-pairing + * add-with-carry instructions we accumulate 32-bit + * values into a 64-bit register. At the end of the + * loop we fold it down to 32-bits and so on. + */ + ba,pt %xcc, 1f + LOAD(prefetch, %o0 + 0x140, #n_reads) + + .align 32 +1: EX_LD(LOAD(lduw, %o0 + 0x00, %o5)) + EX_LD(LOAD(lduw, %o0 + 0x04, %g1)) + EX_LD(LOAD(lduw, %o0 + 0x08, %g2)) + add %o4, %o5, %o4 + EX_ST(STORE(stw, %o5, %o1 + 0x00)) + EX_LD(LOAD(lduw, %o0 + 0x0c, %o5)) + add %o4, %g1, %o4 + EX_ST(STORE(stw, %g1, %o1 + 0x04)) + EX_LD(LOAD(lduw, %o0 + 0x10, %g1)) + add %o4, %g2, %o4 + EX_ST(STORE(stw, %g2, %o1 + 0x08)) + EX_LD(LOAD(lduw, %o0 + 0x14, %g2)) + add %o4, %o5, %o4 + EX_ST(STORE(stw, %o5, %o1 + 0x0c)) + EX_LD(LOAD(lduw, %o0 + 0x18, %o5)) + add %o4, %g1, %o4 + EX_ST(STORE(stw, %g1, %o1 + 0x10)) + EX_LD(LOAD(lduw, %o0 + 0x1c, %g1)) + add %o4, %g2, %o4 + EX_ST(STORE(stw, %g2, %o1 + 0x14)) + EX_LD(LOAD(lduw, %o0 + 0x20, %g2)) + add %o4, %o5, %o4 + EX_ST(STORE(stw, %o5, %o1 + 0x18)) + EX_LD(LOAD(lduw, %o0 + 0x24, %o5)) + add %o4, %g1, %o4 + EX_ST(STORE(stw, %g1, %o1 + 0x1c)) + EX_LD(LOAD(lduw, %o0 + 0x28, %g1)) + add %o4, %g2, %o4 + EX_ST(STORE(stw, %g2, %o1 + 0x20)) + EX_LD(LOAD(lduw, %o0 + 0x2c, %g2)) + add %o4, %o5, %o4 + EX_ST(STORE(stw, %o5, %o1 + 0x24)) + EX_LD(LOAD(lduw, %o0 + 0x30, %o5)) + add %o4, %g1, %o4 + EX_ST(STORE(stw, %g1, %o1 + 0x28)) + EX_LD(LOAD(lduw, %o0 + 0x34, %g1)) + add %o4, %g2, %o4 + EX_ST(STORE(stw, %g2, %o1 + 0x2c)) + EX_LD(LOAD(lduw, %o0 + 0x38, %g2)) + add %o4, %o5, %o4 + EX_ST(STORE(stw, %o5, %o1 + 0x30)) + EX_LD(LOAD(lduw, %o0 + 0x3c, %o5)) + add %o4, %g1, %o4 + EX_ST(STORE(stw, %g1, %o1 + 0x34)) + LOAD(prefetch, %o0 + 0x180, #n_reads) + add %o4, %g2, %o4 + EX_ST(STORE(stw, %g2, %o1 + 0x38)) + subcc %g3, 0x40, %g3 + add %o0, 0x40, %o0 + add %o4, %o5, %o4 + EX_ST(STORE(stw, %o5, %o1 + 0x3c)) + bne,pt %icc, 1b + add %o1, 0x40, %o1 + +2: and %o2, 0x3c, %g3 + brz,pn %g3, 2f + sub %o2, %g3, %o2 +1: EX_LD(LOAD(lduw, %o0 + 0x00, %o5)) + subcc %g3, 0x4, %g3 + add %o0, 0x4, %o0 + add %o4, %o5, %o4 + EX_ST(STORE(stw, %o5, %o1 + 0x00)) + bne,pt %icc, 1b + add %o1, 0x4, %o1 + +2: + /* fold 64-->32 */ + srlx %o4, 32, %o5 + srl %o4, 0, %o4 + add %o4, %o5, %o4 + srlx %o4, 32, %o5 + srl %o4, 0, %o4 + add %o4, %o5, %o4 + + /* fold 32-->16 */ + sethi %hi(0xffff0000), %g1 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + +60: + /* %o4 has the 16-bit sum we have calculated so-far. */ + cmp %o2, 2 + blu,pt %icc, 1f + nop + EX_LD(LOAD(lduh, %o0 + 0x00, %o5)) + sub %o2, 2, %o2 + add %o0, 2, %o0 + add %o4, %o5, %o4 + EX_ST(STORE(sth, %o5, %o1 + 0x00)) + add %o1, 0x2, %o1 +1: brz,pt %o2, 1f + nop + EX_LD(LOAD(ldub, %o0 + 0x00, %o5)) + sub %o2, 1, %o2 + add %o0, 1, %o0 + EX_ST(STORE(stb, %o5, %o1 + 0x00)) + sllx %o5, 8, %o5 + add %o1, 1, %o1 + add %o4, %o5, %o4 +1: + /* fold 32-->16 */ + sethi %hi(0xffff0000), %g1 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + +1: brz,pt GLOBAL_SPARE, 1f + nop + + /* We started with an odd byte, byte-swap the result. */ + srl %o4, 8, %o5 + and %o4, 0xff, %g1 + sll %g1, 8, %g1 + or %o5, %g1, %o4 + +1: addcc %o3, %o4, %o3 + addc %g0, %o3, %o3 + +70: + retl + srl %o3, 0, %o0 + +95: mov 0, GLOBAL_SPARE + brlez,pn %o2, 4f + andcc %o0, 1, %o5 + be,a,pt %icc, 1f + srl %o2, 1, %g1 + sub %o2, 1, %o2 + EX_LD(LOAD(ldub, %o0, GLOBAL_SPARE)) + add %o0, 1, %o0 + EX_ST(STORE(stb, GLOBAL_SPARE, %o1)) + srl %o2, 1, %g1 + add %o1, 1, %o1 +1: brz,a,pn %g1, 3f + andcc %o2, 1, %g0 + andcc %o0, 2, %g0 + be,a,pt %icc, 1f + srl %g1, 1, %g1 + EX_LD(LOAD(lduh, %o0, %o4)) + sub %o2, 2, %o2 + srl %o4, 8, %g2 + sub %g1, 1, %g1 + EX_ST(STORE(stb, %g2, %o1)) + add %o4, GLOBAL_SPARE, GLOBAL_SPARE + EX_ST(STORE(stb, %o4, %o1 + 1)) + add %o0, 2, %o0 + srl %g1, 1, %g1 + add %o1, 2, %o1 +1: brz,a,pn %g1, 2f + andcc %o2, 2, %g0 + EX_LD(LOAD(lduw, %o0, %o4)) +5: srl %o4, 24, %g2 + srl %o4, 16, %g3 + EX_ST(STORE(stb, %g2, %o1)) + srl %o4, 8, %g2 + EX_ST(STORE(stb, %g3, %o1 + 1)) + add %o0, 4, %o0 + EX_ST(STORE(stb, %g2, %o1 + 2)) + addcc %o4, GLOBAL_SPARE, GLOBAL_SPARE + EX_ST(STORE(stb, %o4, %o1 + 3)) + addc GLOBAL_SPARE, %g0, GLOBAL_SPARE + add %o1, 4, %o1 + subcc %g1, 1, %g1 + bne,a,pt %icc, 5b + EX_LD(LOAD(lduw, %o0, %o4)) + sll GLOBAL_SPARE, 16, %g2 + srl GLOBAL_SPARE, 16, GLOBAL_SPARE + srl %g2, 16, %g2 + andcc %o2, 2, %g0 + add %g2, GLOBAL_SPARE, GLOBAL_SPARE +2: be,a,pt %icc, 3f + andcc %o2, 1, %g0 + EX_LD(LOAD(lduh, %o0, %o4)) + andcc %o2, 1, %g0 + srl %o4, 8, %g2 + add %o0, 2, %o0 + EX_ST(STORE(stb, %g2, %o1)) + add GLOBAL_SPARE, %o4, GLOBAL_SPARE + EX_ST(STORE(stb, %o4, %o1 + 1)) + add %o1, 2, %o1 +3: be,a,pt %icc, 1f + sll GLOBAL_SPARE, 16, %o4 + EX_LD(LOAD(ldub, %o0, %g2)) + sll %g2, 8, %o4 + EX_ST(STORE(stb, %g2, %o1)) + add GLOBAL_SPARE, %o4, GLOBAL_SPARE + sll GLOBAL_SPARE, 16, %o4 +1: addcc %o4, GLOBAL_SPARE, GLOBAL_SPARE + srl GLOBAL_SPARE, 16, %o4 + addc %g0, %o4, GLOBAL_SPARE + brz,pt %o5, 4f + srl GLOBAL_SPARE, 8, %o4 + and GLOBAL_SPARE, 0xff, %g2 + and %o4, 0xff, %o4 + sll %g2, 8, %g2 + or %g2, %o4, GLOBAL_SPARE +4: addcc %o3, GLOBAL_SPARE, %o3 + addc %g0, %o3, %o0 + retl + srl %o0, 0, %o0 + .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/csum_copy_from_user.S b/arch/sparc/lib/csum_copy_from_user.S new file mode 100644 index 000000000000..a22eddbe5dba --- /dev/null +++ b/arch/sparc/lib/csum_copy_from_user.S @@ -0,0 +1,21 @@ +/* csum_copy_from_user.S: Checksum+copy from userspace. + * + * Copyright (C) 2005 David S. Miller (davem@davemloft.net) + */ + +#define EX_LD(x) \ +98: x; \ + .section .fixup; \ + .align 4; \ +99: retl; \ + mov -1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + +#define FUNC_NAME __csum_partial_copy_from_user +#define LOAD(type,addr,dest) type##a [addr] %asi, dest + +#include "csum_copy.S" diff --git a/arch/sparc/lib/csum_copy_to_user.S b/arch/sparc/lib/csum_copy_to_user.S new file mode 100644 index 000000000000..d5b12f441f02 --- /dev/null +++ b/arch/sparc/lib/csum_copy_to_user.S @@ -0,0 +1,21 @@ +/* csum_copy_to_user.S: Checksum+copy to userspace. + * + * Copyright (C) 2005 David S. Miller (davem@davemloft.net) + */ + +#define EX_ST(x) \ +98: x; \ + .section .fixup; \ + .align 4; \ +99: retl; \ + mov -1, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + +#define FUNC_NAME __csum_partial_copy_to_user +#define STORE(type,src,addr) type##a src, [addr] %asi + +#include "csum_copy.S" diff --git a/arch/sparc/lib/ipcsum.S b/arch/sparc/lib/ipcsum.S new file mode 100644 index 000000000000..58ca5b9a8778 --- /dev/null +++ b/arch/sparc/lib/ipcsum.S @@ -0,0 +1,34 @@ + .text + .align 32 + .globl ip_fast_csum + .type ip_fast_csum,#function +ip_fast_csum: /* %o0 = iph, %o1 = ihl */ + sub %o1, 4, %g7 + lduw [%o0 + 0x00], %o2 + lduw [%o0 + 0x04], %g2 + lduw [%o0 + 0x08], %g3 + addcc %g2, %o2, %o2 + lduw [%o0 + 0x0c], %g2 + addccc %g3, %o2, %o2 + lduw [%o0 + 0x10], %g3 + + addccc %g2, %o2, %o2 + addc %o2, %g0, %o2 +1: addcc %g3, %o2, %o2 + add %o0, 4, %o0 + addccc %o2, %g0, %o2 + subcc %g7, 1, %g7 + be,a,pt %icc, 2f + sll %o2, 16, %g2 + + lduw [%o0 + 0x10], %g3 + ba,pt %xcc, 1b + nop +2: addcc %o2, %g2, %g2 + srl %g2, 16, %o2 + addc %o2, %g0, %o2 + xnor %g0, %o2, %o2 + set 0xffff, %o1 + retl + and %o2, %o1, %o0 + .size ip_fast_csum, .-ip_fast_csum diff --git a/arch/sparc/lib/mcount.S b/arch/sparc/lib/mcount.S new file mode 100644 index 000000000000..7ce9c65f3592 --- /dev/null +++ b/arch/sparc/lib/mcount.S @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2000 Anton Blanchard (anton@linuxcare.com) + * + * This file implements mcount(), which is used to collect profiling data. + * This can also be tweaked for kernel stack overflow detection. + */ + +#include + +#include +#include + +/* + * This is the main variant and is called by C code. GCC's -pg option + * automatically instruments every C function with a call to this. + */ + +#ifdef CONFIG_STACK_DEBUG + +#define OVSTACKSIZE 4096 /* lets hope this is enough */ + + .data + .align 8 +panicstring: + .asciz "Stack overflow\n" + .align 8 +ovstack: + .skip OVSTACKSIZE +#endif + .text + .align 32 + .globl _mcount + .type _mcount,#function + .globl mcount + .type mcount,#function +_mcount: +mcount: +#ifdef CONFIG_STACK_DEBUG + /* + * Check whether %sp is dangerously low. + */ + ldub [%g6 + TI_FPDEPTH], %g1 + srl %g1, 1, %g3 + add %g3, 1, %g3 + sllx %g3, 8, %g3 ! each fpregs frame is 256b + add %g3, 192, %g3 + add %g6, %g3, %g3 ! where does task_struct+frame end? + sub %g3, STACK_BIAS, %g3 + cmp %sp, %g3 + bg,pt %xcc, 1f + nop + lduh [%g6 + TI_CPU], %g1 + sethi %hi(hardirq_stack), %g3 + or %g3, %lo(hardirq_stack), %g3 + sllx %g1, 3, %g1 + ldx [%g3 + %g1], %g7 + sub %g7, STACK_BIAS, %g7 + cmp %sp, %g7 + bleu,pt %xcc, 2f + sethi %hi(THREAD_SIZE), %g3 + add %g7, %g3, %g7 + cmp %sp, %g7 + blu,pn %xcc, 1f +2: sethi %hi(softirq_stack), %g3 + or %g3, %lo(softirq_stack), %g3 + ldx [%g3 + %g1], %g7 + cmp %sp, %g7 + bleu,pt %xcc, 2f + sethi %hi(THREAD_SIZE), %g3 + add %g7, %g3, %g7 + cmp %sp, %g7 + blu,pn %xcc, 1f + nop + /* If we are already on ovstack, don't hop onto it + * again, we are already trying to output the stack overflow + * message. + */ + sethi %hi(ovstack), %g7 ! cant move to panic stack fast enough + or %g7, %lo(ovstack), %g7 + add %g7, OVSTACKSIZE, %g3 + sub %g3, STACK_BIAS + 192, %g3 + sub %g7, STACK_BIAS, %g7 + cmp %sp, %g7 + blu,pn %xcc, 2f + cmp %sp, %g3 + bleu,pn %xcc, 1f + nop +2: mov %g3, %sp + sethi %hi(panicstring), %g3 + call prom_printf + or %g3, %lo(panicstring), %o0 + call prom_halt + nop +1: +#endif +#ifdef CONFIG_FUNCTION_TRACER +#ifdef CONFIG_DYNAMIC_FTRACE + mov %o7, %o0 + .globl mcount_call +mcount_call: + call ftrace_stub + mov %o0, %o7 +#else + sethi %hi(ftrace_trace_function), %g1 + sethi %hi(ftrace_stub), %g2 + ldx [%g1 + %lo(ftrace_trace_function)], %g1 + or %g2, %lo(ftrace_stub), %g2 + cmp %g1, %g2 + be,pn %icc, 1f + mov %i7, %o1 + jmpl %g1, %g0 + mov %o7, %o0 + /* not reached */ +1: +#endif +#endif + retl + nop + .size _mcount,.-_mcount + .size mcount,.-mcount + +#ifdef CONFIG_FUNCTION_TRACER + .globl ftrace_stub + .type ftrace_stub,#function +ftrace_stub: + retl + nop + .size ftrace_stub,.-ftrace_stub +#ifdef CONFIG_DYNAMIC_FTRACE + .globl ftrace_caller + .type ftrace_caller,#function +ftrace_caller: + mov %i7, %o1 + mov %o7, %o0 + .globl ftrace_call +ftrace_call: + call ftrace_stub + mov %o0, %o7 + retl + nop + .size ftrace_caller,.-ftrace_caller +#endif +#endif diff --git a/arch/sparc/lib/memcmp_64.S b/arch/sparc/lib/memcmp_64.S new file mode 100644 index 000000000000..d3fdaa898566 --- /dev/null +++ b/arch/sparc/lib/memcmp_64.S @@ -0,0 +1,28 @@ +/* + * Sparc64 optimized memcmp code. + * + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Copyright (C) 2000 David S. Miller (davem@redhat.com) + */ + + .text + .align 32 + .globl __memcmp, memcmp +__memcmp: +memcmp: + cmp %o2, 0 ! IEU1 Group +loop: be,pn %icc, ret_0 ! CTI + nop ! IEU0 + ldub [%o0], %g7 ! LSU Group + ldub [%o1], %g3 ! LSU Group + sub %o2, 1, %o2 ! IEU0 + add %o0, 1, %o0 ! IEU1 + add %o1, 1, %o1 ! IEU0 Group + subcc %g7, %g3, %g3 ! IEU1 Group + be,pt %icc, loop ! CTI + cmp %o2, 0 ! IEU1 Group + +ret_n0: retl + mov %g3, %o0 +ret_0: retl + mov 0, %o0 diff --git a/arch/sparc/lib/memmove.S b/arch/sparc/lib/memmove.S new file mode 100644 index 000000000000..97395802c23c --- /dev/null +++ b/arch/sparc/lib/memmove.S @@ -0,0 +1,31 @@ +/* memmove.S: Simple memmove implementation. + * + * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com) + * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz) + */ + + .text + .align 32 + .globl memmove + .type memmove,#function +memmove: /* o0=dst o1=src o2=len */ + mov %o0, %g1 + cmp %o0, %o1 + bleu,pt %xcc, memcpy + add %o1, %o2, %g7 + cmp %g7, %o0 + bleu,pt %xcc, memcpy + add %o0, %o2, %o5 + sub %g7, 1, %o1 + + sub %o5, 1, %o0 +1: ldub [%o1], %g7 + subcc %o2, 1, %o2 + sub %o1, 1, %o1 + stb %g7, [%o0] + bne,pt %icc, 1b + sub %o0, 1, %o0 + + retl + mov %g1, %o0 + .size memmove, .-memmove diff --git a/arch/sparc/lib/memscan_64.S b/arch/sparc/lib/memscan_64.S new file mode 100644 index 000000000000..5686dfa5dc15 --- /dev/null +++ b/arch/sparc/lib/memscan_64.S @@ -0,0 +1,129 @@ +/* + * memscan.S: Optimized memscan for Sparc64. + * + * Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz) + * Copyright (C) 1998 David S. Miller (davem@redhat.com) + */ + +#define HI_MAGIC 0x8080808080808080 +#define LO_MAGIC 0x0101010101010101 +#define ASI_PL 0x88 + + .text + .align 32 + .globl __memscan_zero, __memscan_generic + .globl memscan + +__memscan_zero: + /* %o0 = bufp, %o1 = size */ + brlez,pn %o1, szzero + andcc %o0, 7, %g0 + be,pt %icc, we_are_aligned + sethi %hi(HI_MAGIC), %o4 + ldub [%o0], %o5 +1: subcc %o1, 1, %o1 + brz,pn %o5, 10f + add %o0, 1, %o0 + + be,pn %xcc, szzero + andcc %o0, 7, %g0 + bne,a,pn %icc, 1b + ldub [%o0], %o5 +we_are_aligned: + ldxa [%o0] ASI_PL, %o5 + or %o4, %lo(HI_MAGIC), %o3 + sllx %o3, 32, %o4 + or %o4, %o3, %o3 + + srlx %o3, 7, %o2 +msloop: + sub %o1, 8, %o1 + add %o0, 8, %o0 + sub %o5, %o2, %o4 + xor %o4, %o5, %o4 + andcc %o4, %o3, %g3 + bne,pn %xcc, check_bytes + srlx %o4, 32, %g3 + + brgz,a,pt %o1, msloop + ldxa [%o0] ASI_PL, %o5 +check_bytes: + bne,a,pn %icc, 2f + andcc %o5, 0xff, %g0 + add %o0, -5, %g2 + ba,pt %xcc, 3f + srlx %o5, 32, %g7 + +2: srlx %o5, 8, %g7 + be,pn %icc, 1f + add %o0, -8, %g2 + andcc %g7, 0xff, %g0 + srlx %g7, 8, %g7 + be,pn %icc, 1f + inc %g2 + andcc %g7, 0xff, %g0 + + srlx %g7, 8, %g7 + be,pn %icc, 1f + inc %g2 + andcc %g7, 0xff, %g0 + srlx %g7, 8, %g7 + be,pn %icc, 1f + inc %g2 + andcc %g3, %o3, %g0 + + be,a,pn %icc, 2f + mov %o0, %g2 +3: andcc %g7, 0xff, %g0 + srlx %g7, 8, %g7 + be,pn %icc, 1f + inc %g2 + andcc %g7, 0xff, %g0 + srlx %g7, 8, %g7 + + be,pn %icc, 1f + inc %g2 + andcc %g7, 0xff, %g0 + srlx %g7, 8, %g7 + be,pn %icc, 1f + inc %g2 + andcc %g7, 0xff, %g0 + srlx %g7, 8, %g7 + + be,pn %icc, 1f + inc %g2 +2: brgz,a,pt %o1, msloop + ldxa [%o0] ASI_PL, %o5 + inc %g2 +1: add %o0, %o1, %o0 + cmp %g2, %o0 + retl + + movle %xcc, %g2, %o0 +10: retl + sub %o0, 1, %o0 +szzero: retl + nop + +memscan: +__memscan_generic: + /* %o0 = addr, %o1 = c, %o2 = size */ + brz,pn %o2, 3f + add %o0, %o2, %o3 + ldub [%o0], %o5 + sub %g0, %o2, %o4 +1: + cmp %o5, %o1 + be,pn %icc, 2f + addcc %o4, 1, %o4 + bne,a,pt %xcc, 1b + ldub [%o3 + %o4], %o5 + retl + /* The delay slot is the same as the next insn, this is just to make it look more awful */ +2: + add %o3, %o4, %o0 + retl + sub %o0, 1, %o0 +3: + retl + nop diff --git a/arch/sparc/lib/rwsem_64.S b/arch/sparc/lib/rwsem_64.S new file mode 100644 index 000000000000..91a7d29a79d5 --- /dev/null +++ b/arch/sparc/lib/rwsem_64.S @@ -0,0 +1,163 @@ +/* rwsem.S: RW semaphore assembler. + * + * Written by David S. Miller (davem@redhat.com), 2001. + * Derived from asm-i386/rwsem.h + */ + +#include + + .section .sched.text, "ax" + + .globl __down_read +__down_read: +1: lduw [%o0], %g1 + add %g1, 1, %g7 + cas [%o0], %g1, %g7 + cmp %g1, %g7 + bne,pn %icc, 1b + add %g7, 1, %g7 + cmp %g7, 0 + bl,pn %icc, 3f + nop +2: + retl + nop +3: + save %sp, -192, %sp + call rwsem_down_read_failed + mov %i0, %o0 + ret + restore + .size __down_read, .-__down_read + + .globl __down_read_trylock +__down_read_trylock: +1: lduw [%o0], %g1 + add %g1, 1, %g7 + cmp %g7, 0 + bl,pn %icc, 2f + mov 0, %o1 + cas [%o0], %g1, %g7 + cmp %g1, %g7 + bne,pn %icc, 1b + mov 1, %o1 +2: retl + mov %o1, %o0 + .size __down_read_trylock, .-__down_read_trylock + + .globl __down_write +__down_write: + sethi %hi(RWSEM_ACTIVE_WRITE_BIAS), %g1 + or %g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1 +1: + lduw [%o0], %g3 + add %g3, %g1, %g7 + cas [%o0], %g3, %g7 + cmp %g3, %g7 + bne,pn %icc, 1b + cmp %g7, 0 + bne,pn %icc, 3f + nop +2: retl + nop +3: + save %sp, -192, %sp + call rwsem_down_write_failed + mov %i0, %o0 + ret + restore + .size __down_write, .-__down_write + + .globl __down_write_trylock +__down_write_trylock: + sethi %hi(RWSEM_ACTIVE_WRITE_BIAS), %g1 + or %g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1 +1: + lduw [%o0], %g3 + cmp %g3, 0 + bne,pn %icc, 2f + mov 0, %o1 + add %g3, %g1, %g7 + cas [%o0], %g3, %g7 + cmp %g3, %g7 + bne,pn %icc, 1b + mov 1, %o1 +2: retl + mov %o1, %o0 + .size __down_write_trylock, .-__down_write_trylock + + .globl __up_read +__up_read: +1: + lduw [%o0], %g1 + sub %g1, 1, %g7 + cas [%o0], %g1, %g7 + cmp %g1, %g7 + bne,pn %icc, 1b + cmp %g7, 0 + bl,pn %icc, 3f + nop +2: retl + nop +3: sethi %hi(RWSEM_ACTIVE_MASK), %g1 + sub %g7, 1, %g7 + or %g1, %lo(RWSEM_ACTIVE_MASK), %g1 + andcc %g7, %g1, %g0 + bne,pn %icc, 2b + nop + save %sp, -192, %sp + call rwsem_wake + mov %i0, %o0 + ret + restore + .size __up_read, .-__up_read + + .globl __up_write +__up_write: + sethi %hi(RWSEM_ACTIVE_WRITE_BIAS), %g1 + or %g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1 +1: + lduw [%o0], %g3 + sub %g3, %g1, %g7 + cas [%o0], %g3, %g7 + cmp %g3, %g7 + bne,pn %icc, 1b + sub %g7, %g1, %g7 + cmp %g7, 0 + bl,pn %icc, 3f + nop +2: + retl + nop +3: + save %sp, -192, %sp + call rwsem_wake + mov %i0, %o0 + ret + restore + .size __up_write, .-__up_write + + .globl __downgrade_write +__downgrade_write: + sethi %hi(RWSEM_WAITING_BIAS), %g1 + or %g1, %lo(RWSEM_WAITING_BIAS), %g1 +1: + lduw [%o0], %g3 + sub %g3, %g1, %g7 + cas [%o0], %g3, %g7 + cmp %g3, %g7 + bne,pn %icc, 1b + sub %g7, %g1, %g7 + cmp %g7, 0 + bl,pn %icc, 3f + nop +2: + retl + nop +3: + save %sp, -192, %sp + call rwsem_downgrade_wake + mov %i0, %o0 + ret + restore + .size __downgrade_write, .-__downgrade_write diff --git a/arch/sparc/lib/strlen_64.S b/arch/sparc/lib/strlen_64.S new file mode 100644 index 000000000000..e9ba1920d818 --- /dev/null +++ b/arch/sparc/lib/strlen_64.S @@ -0,0 +1,80 @@ +/* strlen.S: Sparc64 optimized strlen code + * Hand optimized from GNU libc's strlen + * Copyright (C) 1991,1996 Free Software Foundation + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996, 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#define LO_MAGIC 0x01010101 +#define HI_MAGIC 0x80808080 + + .align 32 + .globl strlen + .type strlen,#function +strlen: + mov %o0, %o1 + andcc %o0, 3, %g0 + be,pt %icc, 9f + sethi %hi(HI_MAGIC), %o4 + ldub [%o0], %o5 + brz,pn %o5, 11f + add %o0, 1, %o0 + andcc %o0, 3, %g0 + be,pn %icc, 4f + or %o4, %lo(HI_MAGIC), %o3 + ldub [%o0], %o5 + brz,pn %o5, 12f + add %o0, 1, %o0 + andcc %o0, 3, %g0 + be,pt %icc, 5f + sethi %hi(LO_MAGIC), %o4 + ldub [%o0], %o5 + brz,pn %o5, 13f + add %o0, 1, %o0 + ba,pt %icc, 8f + or %o4, %lo(LO_MAGIC), %o2 +9: + or %o4, %lo(HI_MAGIC), %o3 +4: + sethi %hi(LO_MAGIC), %o4 +5: + or %o4, %lo(LO_MAGIC), %o2 +8: + ld [%o0], %o5 +2: + sub %o5, %o2, %o4 + andcc %o4, %o3, %g0 + be,pt %icc, 8b + add %o0, 4, %o0 + + /* Check every byte. */ + srl %o5, 24, %g7 + andcc %g7, 0xff, %g0 + be,pn %icc, 1f + add %o0, -4, %o4 + srl %o5, 16, %g7 + andcc %g7, 0xff, %g0 + be,pn %icc, 1f + add %o4, 1, %o4 + srl %o5, 8, %g7 + andcc %g7, 0xff, %g0 + be,pn %icc, 1f + add %o4, 1, %o4 + andcc %o5, 0xff, %g0 + bne,a,pt %icc, 2b + ld [%o0], %o5 + add %o4, 1, %o4 +1: + retl + sub %o4, %o1, %o0 +11: + retl + mov 0, %o0 +12: + retl + mov 1, %o0 +13: + retl + mov 2, %o0 + + .size strlen, .-strlen diff --git a/arch/sparc/lib/strlen_user_64.S b/arch/sparc/lib/strlen_user_64.S new file mode 100644 index 000000000000..114ed111e251 --- /dev/null +++ b/arch/sparc/lib/strlen_user_64.S @@ -0,0 +1,95 @@ +/* strlen_user.S: Sparc64 optimized strlen_user code + * + * Return length of string in userspace including terminating 0 + * or 0 for error + * + * Copyright (C) 1991,1996 Free Software Foundation + * Copyright (C) 1996,1999 David S. Miller (davem@redhat.com) + * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include + +#define LO_MAGIC 0x01010101 +#define HI_MAGIC 0x80808080 + + .align 4 + .global __strlen_user, __strnlen_user +__strlen_user: + sethi %hi(32768), %o1 +__strnlen_user: + mov %o1, %g1 + mov %o0, %o1 + andcc %o0, 3, %g0 + be,pt %icc, 9f + sethi %hi(HI_MAGIC), %o4 +10: lduba [%o0] %asi, %o5 + brz,pn %o5, 21f + add %o0, 1, %o0 + andcc %o0, 3, %g0 + be,pn %icc, 4f + or %o4, %lo(HI_MAGIC), %o3 +11: lduba [%o0] %asi, %o5 + brz,pn %o5, 22f + add %o0, 1, %o0 + andcc %o0, 3, %g0 + be,pt %icc, 13f + srl %o3, 7, %o2 +12: lduba [%o0] %asi, %o5 + brz,pn %o5, 23f + add %o0, 1, %o0 + ba,pt %icc, 2f +15: lda [%o0] %asi, %o5 +9: or %o4, %lo(HI_MAGIC), %o3 +4: srl %o3, 7, %o2 +13: lda [%o0] %asi, %o5 +2: sub %o5, %o2, %o4 + andcc %o4, %o3, %g0 + bne,pn %icc, 82f + add %o0, 4, %o0 + sub %o0, %o1, %g2 +81: cmp %g2, %g1 + blu,pt %icc, 13b + mov %o0, %o4 + ba,a,pt %xcc, 1f + + /* Check every byte. */ +82: srl %o5, 24, %g7 + andcc %g7, 0xff, %g0 + be,pn %icc, 1f + add %o0, -3, %o4 + srl %o5, 16, %g7 + andcc %g7, 0xff, %g0 + be,pn %icc, 1f + add %o4, 1, %o4 + srl %o5, 8, %g7 + andcc %g7, 0xff, %g0 + be,pn %icc, 1f + add %o4, 1, %o4 + andcc %o5, 0xff, %g0 + bne,pt %icc, 81b + sub %o0, %o1, %g2 + add %o4, 1, %o4 +1: retl + sub %o4, %o1, %o0 +21: retl + mov 1, %o0 +22: retl + mov 2, %o0 +23: retl + mov 3, %o0 + + .section .fixup,#alloc,#execinstr + .align 4 +30: + retl + clr %o0 + + .section __ex_table,"a" + .align 4 + + .word 10b, 30b + .word 11b, 30b + .word 12b, 30b + .word 15b, 30b + .word 13b, 30b diff --git a/arch/sparc/lib/strncmp_64.S b/arch/sparc/lib/strncmp_64.S new file mode 100644 index 000000000000..980e83751556 --- /dev/null +++ b/arch/sparc/lib/strncmp_64.S @@ -0,0 +1,32 @@ +/* + * Sparc64 optimized strncmp code. + * + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include + + .text + .align 32 + .globl strncmp + .type strncmp,#function +strncmp: + brlez,pn %o2, 3f + lduba [%o0] (ASI_PNF), %o3 +1: + add %o0, 1, %o0 + ldub [%o1], %o4 + brz,pn %o3, 2f + add %o1, 1, %o1 + cmp %o3, %o4 + bne,pn %icc, 2f + subcc %o2, 1, %o2 + bne,a,pt %xcc, 1b + ldub [%o0], %o3 +2: + retl + sub %o3, %o4, %o0 +3: + retl + clr %o0 + .size strncmp, .-strncmp diff --git a/arch/sparc/lib/strncpy_from_user_64.S b/arch/sparc/lib/strncpy_from_user_64.S new file mode 100644 index 000000000000..511c8f136f95 --- /dev/null +++ b/arch/sparc/lib/strncpy_from_user_64.S @@ -0,0 +1,135 @@ +/* + * strncpy_from_user.S: Sparc64 strncpy from userspace. + * + * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz) + */ + +#include +#include + + .data + .align 8 +0: .xword 0x0101010101010101 + + .text + .align 32 + + /* Must return: + * + * -EFAULT for an exception + * count if we hit the buffer limit + * bytes copied if we hit a null byte + * (without the null byte) + * + * This implementation assumes: + * %o1 is 8 aligned => !(%o2 & 7) + * %o0 is 8 aligned (if not, it will be slooooow, but will work) + * + * This is optimized for the common case: + * in my stats, 90% of src are 8 aligned (even on sparc32) + * and average length is 18 or so. + */ + + .globl __strncpy_from_user + .type __strncpy_from_user,#function +__strncpy_from_user: + /* %o0=dest, %o1=src, %o2=count */ + andcc %o1, 7, %g0 ! IEU1 Group + bne,pn %icc, 30f ! CTI + add %o0, %o2, %g3 ! IEU0 +60: ldxa [%o1] %asi, %g1 ! Load Group + brlez,pn %o2, 10f ! CTI + mov %o0, %o3 ! IEU0 +50: sethi %hi(0b), %o4 ! IEU0 Group + ldx [%o4 + %lo(0b)], %o4 ! Load + sllx %o4, 7, %o5 ! IEU1 Group +1: sub %g1, %o4, %g2 ! IEU0 Group + stx %g1, [%o0] ! Store + add %o0, 8, %o0 ! IEU1 + andcc %g2, %o5, %g0 ! IEU1 Group + bne,pn %xcc, 5f ! CTI + add %o1, 8, %o1 ! IEU0 + cmp %o0, %g3 ! IEU1 Group + bl,a,pt %xcc, 1b ! CTI +61: ldxa [%o1] %asi, %g1 ! Load +10: retl ! CTI Group + mov %o2, %o0 ! IEU0 +5: srlx %g2, 32, %g7 ! IEU0 Group + sethi %hi(0xff00), %o4 ! IEU1 + andcc %g7, %o5, %g0 ! IEU1 Group + be,pn %icc, 2f ! CTI + or %o4, %lo(0xff00), %o4 ! IEU0 + srlx %g1, 48, %g7 ! IEU0 Group + andcc %g7, %o4, %g0 ! IEU1 Group + be,pn %icc, 50f ! CTI + andcc %g7, 0xff, %g0 ! IEU1 Group + be,pn %icc, 51f ! CTI + srlx %g1, 32, %g7 ! IEU0 + andcc %g7, %o4, %g0 ! IEU1 Group + be,pn %icc, 52f ! CTI + andcc %g7, 0xff, %g0 ! IEU1 Group + be,pn %icc, 53f ! CTI +2: andcc %g2, %o5, %g0 ! IEU1 Group + be,pn %icc, 2f ! CTI + srl %g1, 16, %g7 ! IEU0 + andcc %g7, %o4, %g0 ! IEU1 Group + be,pn %icc, 54f ! CTI + andcc %g7, 0xff, %g0 ! IEU1 Group + be,pn %icc, 55f ! CTI + andcc %g1, %o4, %g0 ! IEU1 Group + be,pn %icc, 56f ! CTI + andcc %g1, 0xff, %g0 ! IEU1 Group + be,a,pn %icc, 57f ! CTI + sub %o0, %o3, %o0 ! IEU0 +2: cmp %o0, %g3 ! IEU1 Group + bl,a,pt %xcc, 50b ! CTI +62: ldxa [%o1] %asi, %g1 ! Load + retl ! CTI Group + mov %o2, %o0 ! IEU0 +50: sub %o0, %o3, %o0 + retl + sub %o0, 8, %o0 +51: sub %o0, %o3, %o0 + retl + sub %o0, 7, %o0 +52: sub %o0, %o3, %o0 + retl + sub %o0, 6, %o0 +53: sub %o0, %o3, %o0 + retl + sub %o0, 5, %o0 +54: sub %o0, %o3, %o0 + retl + sub %o0, 4, %o0 +55: sub %o0, %o3, %o0 + retl + sub %o0, 3, %o0 +56: sub %o0, %o3, %o0 + retl + sub %o0, 2, %o0 +57: retl + sub %o0, 1, %o0 +30: brlez,pn %o2, 3f + sub %g0, %o2, %o3 + add %o0, %o2, %o0 +63: lduba [%o1] %asi, %o4 +1: add %o1, 1, %o1 + brz,pn %o4, 2f + stb %o4, [%o0 + %o3] + addcc %o3, 1, %o3 + bne,pt %xcc, 1b +64: lduba [%o1] %asi, %o4 +3: retl + mov %o2, %o0 +2: retl + add %o2, %o3, %o0 + .size __strncpy_from_user, .-__strncpy_from_user + + .section __ex_table,"a" + .align 4 + .word 60b, __retl_efault + .word 61b, __retl_efault + .word 62b, __retl_efault + .word 63b, __retl_efault + .word 64b, __retl_efault + .previous diff --git a/arch/sparc/lib/user_fixup.c b/arch/sparc/lib/user_fixup.c new file mode 100644 index 000000000000..05a361b0a1a4 --- /dev/null +++ b/arch/sparc/lib/user_fixup.c @@ -0,0 +1,66 @@ +/* user_fixup.c: Fix up user copy faults. + * + * Copyright (C) 2004 David S. Miller + */ + +#include +#include +#include +#include +#include + +/* Calculating the exact fault address when using + * block loads and stores can be very complicated. + * + * Instead of trying to be clever and handling all + * of the cases, just fix things up simply here. + */ + +static unsigned long compute_size(unsigned long start, unsigned long size, unsigned long *offset) +{ + unsigned long fault_addr = current_thread_info()->fault_address; + unsigned long end = start + size; + + if (fault_addr < start || fault_addr >= end) { + *offset = 0; + } else { + *offset = fault_addr - start; + size = end - fault_addr; + } + return size; +} + +unsigned long copy_from_user_fixup(void *to, const void __user *from, unsigned long size) +{ + unsigned long offset; + + size = compute_size((unsigned long) from, size, &offset); + if (likely(size)) + memset(to + offset, 0, size); + + return size; +} + +unsigned long copy_to_user_fixup(void __user *to, const void *from, unsigned long size) +{ + unsigned long offset; + + return compute_size((unsigned long) to, size, &offset); +} + +unsigned long copy_in_user_fixup(void __user *to, void __user *from, unsigned long size) +{ + unsigned long fault_addr = current_thread_info()->fault_address; + unsigned long start = (unsigned long) to; + unsigned long end = start + size; + + if (fault_addr >= start && fault_addr < end) + return end - fault_addr; + + start = (unsigned long) from; + end = start + size; + if (fault_addr >= start && fault_addr < end) + return end - fault_addr; + + return size; +} diff --git a/arch/sparc/lib/xor.S b/arch/sparc/lib/xor.S new file mode 100644 index 000000000000..f44f58f40234 --- /dev/null +++ b/arch/sparc/lib/xor.S @@ -0,0 +1,652 @@ +/* + * arch/sparc64/lib/xor.S + * + * High speed xor_block operation for RAID4/5 utilizing the + * UltraSparc Visual Instruction Set and Niagara store-init/twin-load. + * + * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz) + * Copyright (C) 2006 David S. Miller + */ + +#include +#include +#include +#include + +/* + * Requirements: + * !(((long)dest | (long)sourceN) & (64 - 1)) && + * !(len & 127) && len >= 256 + */ + .text + .align 32 + + /* VIS versions. */ + .globl xor_vis_2 + .type xor_vis_2,#function +xor_vis_2: + rd %fprs, %o5 + andcc %o5, FPRS_FEF|FPRS_DU, %g0 + be,pt %icc, 0f + sethi %hi(VISenter), %g1 + jmpl %g1 + %lo(VISenter), %g7 + add %g7, 8, %g7 +0: wr %g0, FPRS_FEF, %fprs + rd %asi, %g1 + wr %g0, ASI_BLK_P, %asi + membar #LoadStore|#StoreLoad|#StoreStore + sub %o0, 128, %o0 + ldda [%o1] %asi, %f0 + ldda [%o2] %asi, %f16 + +2: ldda [%o1 + 64] %asi, %f32 + fxor %f0, %f16, %f16 + fxor %f2, %f18, %f18 + fxor %f4, %f20, %f20 + fxor %f6, %f22, %f22 + fxor %f8, %f24, %f24 + fxor %f10, %f26, %f26 + fxor %f12, %f28, %f28 + fxor %f14, %f30, %f30 + stda %f16, [%o1] %asi + ldda [%o2 + 64] %asi, %f48 + ldda [%o1 + 128] %asi, %f0 + fxor %f32, %f48, %f48 + fxor %f34, %f50, %f50 + add %o1, 128, %o1 + fxor %f36, %f52, %f52 + add %o2, 128, %o2 + fxor %f38, %f54, %f54 + subcc %o0, 128, %o0 + fxor %f40, %f56, %f56 + fxor %f42, %f58, %f58 + fxor %f44, %f60, %f60 + fxor %f46, %f62, %f62 + stda %f48, [%o1 - 64] %asi + bne,pt %xcc, 2b + ldda [%o2] %asi, %f16 + + ldda [%o1 + 64] %asi, %f32 + fxor %f0, %f16, %f16 + fxor %f2, %f18, %f18 + fxor %f4, %f20, %f20 + fxor %f6, %f22, %f22 + fxor %f8, %f24, %f24 + fxor %f10, %f26, %f26 + fxor %f12, %f28, %f28 + fxor %f14, %f30, %f30 + stda %f16, [%o1] %asi + ldda [%o2 + 64] %asi, %f48 + membar #Sync + fxor %f32, %f48, %f48 + fxor %f34, %f50, %f50 + fxor %f36, %f52, %f52 + fxor %f38, %f54, %f54 + fxor %f40, %f56, %f56 + fxor %f42, %f58, %f58 + fxor %f44, %f60, %f60 + fxor %f46, %f62, %f62 + stda %f48, [%o1 + 64] %asi + membar #Sync|#StoreStore|#StoreLoad + wr %g1, %g0, %asi + retl + wr %g0, 0, %fprs + .size xor_vis_2, .-xor_vis_2 + + .globl xor_vis_3 + .type xor_vis_3,#function +xor_vis_3: + rd %fprs, %o5 + andcc %o5, FPRS_FEF|FPRS_DU, %g0 + be,pt %icc, 0f + sethi %hi(VISenter), %g1 + jmpl %g1 + %lo(VISenter), %g7 + add %g7, 8, %g7 +0: wr %g0, FPRS_FEF, %fprs + rd %asi, %g1 + wr %g0, ASI_BLK_P, %asi + membar #LoadStore|#StoreLoad|#StoreStore + sub %o0, 64, %o0 + ldda [%o1] %asi, %f0 + ldda [%o2] %asi, %f16 + +3: ldda [%o3] %asi, %f32 + fxor %f0, %f16, %f48 + fxor %f2, %f18, %f50 + add %o1, 64, %o1 + fxor %f4, %f20, %f52 + fxor %f6, %f22, %f54 + add %o2, 64, %o2 + fxor %f8, %f24, %f56 + fxor %f10, %f26, %f58 + fxor %f12, %f28, %f60 + fxor %f14, %f30, %f62 + ldda [%o1] %asi, %f0 + fxor %f48, %f32, %f48 + fxor %f50, %f34, %f50 + fxor %f52, %f36, %f52 + fxor %f54, %f38, %f54 + add %o3, 64, %o3 + fxor %f56, %f40, %f56 + fxor %f58, %f42, %f58 + subcc %o0, 64, %o0 + fxor %f60, %f44, %f60 + fxor %f62, %f46, %f62 + stda %f48, [%o1 - 64] %asi + bne,pt %xcc, 3b + ldda [%o2] %asi, %f16 + + ldda [%o3] %asi, %f32 + fxor %f0, %f16, %f48 + fxor %f2, %f18, %f50 + fxor %f4, %f20, %f52 + fxor %f6, %f22, %f54 + fxor %f8, %f24, %f56 + fxor %f10, %f26, %f58 + fxor %f12, %f28, %f60 + fxor %f14, %f30, %f62 + membar #Sync + fxor %f48, %f32, %f48 + fxor %f50, %f34, %f50 + fxor %f52, %f36, %f52 + fxor %f54, %f38, %f54 + fxor %f56, %f40, %f56 + fxor %f58, %f42, %f58 + fxor %f60, %f44, %f60 + fxor %f62, %f46, %f62 + stda %f48, [%o1] %asi + membar #Sync|#StoreStore|#StoreLoad + wr %g1, %g0, %asi + retl + wr %g0, 0, %fprs + .size xor_vis_3, .-xor_vis_3 + + .globl xor_vis_4 + .type xor_vis_4,#function +xor_vis_4: + rd %fprs, %o5 + andcc %o5, FPRS_FEF|FPRS_DU, %g0 + be,pt %icc, 0f + sethi %hi(VISenter), %g1 + jmpl %g1 + %lo(VISenter), %g7 + add %g7, 8, %g7 +0: wr %g0, FPRS_FEF, %fprs + rd %asi, %g1 + wr %g0, ASI_BLK_P, %asi + membar #LoadStore|#StoreLoad|#StoreStore + sub %o0, 64, %o0 + ldda [%o1] %asi, %f0 + ldda [%o2] %asi, %f16 + +4: ldda [%o3] %asi, %f32 + fxor %f0, %f16, %f16 + fxor %f2, %f18, %f18 + add %o1, 64, %o1 + fxor %f4, %f20, %f20 + fxor %f6, %f22, %f22 + add %o2, 64, %o2 + fxor %f8, %f24, %f24 + fxor %f10, %f26, %f26 + fxor %f12, %f28, %f28 + fxor %f14, %f30, %f30 + ldda [%o4] %asi, %f48 + fxor %f16, %f32, %f32 + fxor %f18, %f34, %f34 + fxor %f20, %f36, %f36 + fxor %f22, %f38, %f38 + add %o3, 64, %o3 + fxor %f24, %f40, %f40 + fxor %f26, %f42, %f42 + fxor %f28, %f44, %f44 + fxor %f30, %f46, %f46 + ldda [%o1] %asi, %f0 + fxor %f32, %f48, %f48 + fxor %f34, %f50, %f50 + fxor %f36, %f52, %f52 + add %o4, 64, %o4 + fxor %f38, %f54, %f54 + fxor %f40, %f56, %f56 + fxor %f42, %f58, %f58 + subcc %o0, 64, %o0 + fxor %f44, %f60, %f60 + fxor %f46, %f62, %f62 + stda %f48, [%o1 - 64] %asi + bne,pt %xcc, 4b + ldda [%o2] %asi, %f16 + + ldda [%o3] %asi, %f32 + fxor %f0, %f16, %f16 + fxor %f2, %f18, %f18 + fxor %f4, %f20, %f20 + fxor %f6, %f22, %f22 + fxor %f8, %f24, %f24 + fxor %f10, %f26, %f26 + fxor %f12, %f28, %f28 + fxor %f14, %f30, %f30 + ldda [%o4] %asi, %f48 + fxor %f16, %f32, %f32 + fxor %f18, %f34, %f34 + fxor %f20, %f36, %f36 + fxor %f22, %f38, %f38 + fxor %f24, %f40, %f40 + fxor %f26, %f42, %f42 + fxor %f28, %f44, %f44 + fxor %f30, %f46, %f46 + membar #Sync + fxor %f32, %f48, %f48 + fxor %f34, %f50, %f50 + fxor %f36, %f52, %f52 + fxor %f38, %f54, %f54 + fxor %f40, %f56, %f56 + fxor %f42, %f58, %f58 + fxor %f44, %f60, %f60 + fxor %f46, %f62, %f62 + stda %f48, [%o1] %asi + membar #Sync|#StoreStore|#StoreLoad + wr %g1, %g0, %asi + retl + wr %g0, 0, %fprs + .size xor_vis_4, .-xor_vis_4 + + .globl xor_vis_5 + .type xor_vis_5,#function +xor_vis_5: + save %sp, -192, %sp + rd %fprs, %o5 + andcc %o5, FPRS_FEF|FPRS_DU, %g0 + be,pt %icc, 0f + sethi %hi(VISenter), %g1 + jmpl %g1 + %lo(VISenter), %g7 + add %g7, 8, %g7 +0: wr %g0, FPRS_FEF, %fprs + rd %asi, %g1 + wr %g0, ASI_BLK_P, %asi + membar #LoadStore|#StoreLoad|#StoreStore + sub %i0, 64, %i0 + ldda [%i1] %asi, %f0 + ldda [%i2] %asi, %f16 + +5: ldda [%i3] %asi, %f32 + fxor %f0, %f16, %f48 + fxor %f2, %f18, %f50 + add %i1, 64, %i1 + fxor %f4, %f20, %f52 + fxor %f6, %f22, %f54 + add %i2, 64, %i2 + fxor %f8, %f24, %f56 + fxor %f10, %f26, %f58 + fxor %f12, %f28, %f60 + fxor %f14, %f30, %f62 + ldda [%i4] %asi, %f16 + fxor %f48, %f32, %f48 + fxor %f50, %f34, %f50 + fxor %f52, %f36, %f52 + fxor %f54, %f38, %f54 + add %i3, 64, %i3 + fxor %f56, %f40, %f56 + fxor %f58, %f42, %f58 + fxor %f60, %f44, %f60 + fxor %f62, %f46, %f62 + ldda [%i5] %asi, %f32 + fxor %f48, %f16, %f48 + fxor %f50, %f18, %f50 + add %i4, 64, %i4 + fxor %f52, %f20, %f52 + fxor %f54, %f22, %f54 + add %i5, 64, %i5 + fxor %f56, %f24, %f56 + fxor %f58, %f26, %f58 + fxor %f60, %f28, %f60 + fxor %f62, %f30, %f62 + ldda [%i1] %asi, %f0 + fxor %f48, %f32, %f48 + fxor %f50, %f34, %f50 + fxor %f52, %f36, %f52 + fxor %f54, %f38, %f54 + fxor %f56, %f40, %f56 + fxor %f58, %f42, %f58 + subcc %i0, 64, %i0 + fxor %f60, %f44, %f60 + fxor %f62, %f46, %f62 + stda %f48, [%i1 - 64] %asi + bne,pt %xcc, 5b + ldda [%i2] %asi, %f16 + + ldda [%i3] %asi, %f32 + fxor %f0, %f16, %f48 + fxor %f2, %f18, %f50 + fxor %f4, %f20, %f52 + fxor %f6, %f22, %f54 + fxor %f8, %f24, %f56 + fxor %f10, %f26, %f58 + fxor %f12, %f28, %f60 + fxor %f14, %f30, %f62 + ldda [%i4] %asi, %f16 + fxor %f48, %f32, %f48 + fxor %f50, %f34, %f50 + fxor %f52, %f36, %f52 + fxor %f54, %f38, %f54 + fxor %f56, %f40, %f56 + fxor %f58, %f42, %f58 + fxor %f60, %f44, %f60 + fxor %f62, %f46, %f62 + ldda [%i5] %asi, %f32 + fxor %f48, %f16, %f48 + fxor %f50, %f18, %f50 + fxor %f52, %f20, %f52 + fxor %f54, %f22, %f54 + fxor %f56, %f24, %f56 + fxor %f58, %f26, %f58 + fxor %f60, %f28, %f60 + fxor %f62, %f30, %f62 + membar #Sync + fxor %f48, %f32, %f48 + fxor %f50, %f34, %f50 + fxor %f52, %f36, %f52 + fxor %f54, %f38, %f54 + fxor %f56, %f40, %f56 + fxor %f58, %f42, %f58 + fxor %f60, %f44, %f60 + fxor %f62, %f46, %f62 + stda %f48, [%i1] %asi + membar #Sync|#StoreStore|#StoreLoad + wr %g1, %g0, %asi + wr %g0, 0, %fprs + ret + restore + .size xor_vis_5, .-xor_vis_5 + + /* Niagara versions. */ + .globl xor_niagara_2 + .type xor_niagara_2,#function +xor_niagara_2: /* %o0=bytes, %o1=dest, %o2=src */ + save %sp, -192, %sp + prefetch [%i1], #n_writes + prefetch [%i2], #one_read + rd %asi, %g7 + wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi + srlx %i0, 6, %g1 + mov %i1, %i0 + mov %i2, %i1 +1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src + 0x00 */ + ldda [%i1 + 0x10] %asi, %i4 /* %i4/%i5 = src + 0x10 */ + ldda [%i1 + 0x20] %asi, %g2 /* %g2/%g3 = src + 0x20 */ + ldda [%i1 + 0x30] %asi, %l0 /* %l0/%l1 = src + 0x30 */ + prefetch [%i1 + 0x40], #one_read + ldda [%i0 + 0x00] %asi, %o0 /* %o0/%o1 = dest + 0x00 */ + ldda [%i0 + 0x10] %asi, %o2 /* %o2/%o3 = dest + 0x10 */ + ldda [%i0 + 0x20] %asi, %o4 /* %o4/%o5 = dest + 0x20 */ + ldda [%i0 + 0x30] %asi, %l2 /* %l2/%l3 = dest + 0x30 */ + prefetch [%i0 + 0x40], #n_writes + xor %o0, %i2, %o0 + xor %o1, %i3, %o1 + stxa %o0, [%i0 + 0x00] %asi + stxa %o1, [%i0 + 0x08] %asi + xor %o2, %i4, %o2 + xor %o3, %i5, %o3 + stxa %o2, [%i0 + 0x10] %asi + stxa %o3, [%i0 + 0x18] %asi + xor %o4, %g2, %o4 + xor %o5, %g3, %o5 + stxa %o4, [%i0 + 0x20] %asi + stxa %o5, [%i0 + 0x28] %asi + xor %l2, %l0, %l2 + xor %l3, %l1, %l3 + stxa %l2, [%i0 + 0x30] %asi + stxa %l3, [%i0 + 0x38] %asi + add %i0, 0x40, %i0 + subcc %g1, 1, %g1 + bne,pt %xcc, 1b + add %i1, 0x40, %i1 + membar #Sync + wr %g7, 0x0, %asi + ret + restore + .size xor_niagara_2, .-xor_niagara_2 + + .globl xor_niagara_3 + .type xor_niagara_3,#function +xor_niagara_3: /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2 */ + save %sp, -192, %sp + prefetch [%i1], #n_writes + prefetch [%i2], #one_read + prefetch [%i3], #one_read + rd %asi, %g7 + wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi + srlx %i0, 6, %g1 + mov %i1, %i0 + mov %i2, %i1 + mov %i3, %l7 +1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src1 + 0x00 */ + ldda [%i1 + 0x10] %asi, %i4 /* %i4/%i5 = src1 + 0x10 */ + ldda [%l7 + 0x00] %asi, %g2 /* %g2/%g3 = src2 + 0x00 */ + ldda [%l7 + 0x10] %asi, %l0 /* %l0/%l1 = src2 + 0x10 */ + ldda [%i0 + 0x00] %asi, %o0 /* %o0/%o1 = dest + 0x00 */ + ldda [%i0 + 0x10] %asi, %o2 /* %o2/%o3 = dest + 0x10 */ + xor %g2, %i2, %g2 + xor %g3, %i3, %g3 + xor %o0, %g2, %o0 + xor %o1, %g3, %o1 + stxa %o0, [%i0 + 0x00] %asi + stxa %o1, [%i0 + 0x08] %asi + ldda [%i1 + 0x20] %asi, %i2 /* %i2/%i3 = src1 + 0x20 */ + ldda [%l7 + 0x20] %asi, %g2 /* %g2/%g3 = src2 + 0x20 */ + ldda [%i0 + 0x20] %asi, %o0 /* %o0/%o1 = dest + 0x20 */ + xor %l0, %i4, %l0 + xor %l1, %i5, %l1 + xor %o2, %l0, %o2 + xor %o3, %l1, %o3 + stxa %o2, [%i0 + 0x10] %asi + stxa %o3, [%i0 + 0x18] %asi + ldda [%i1 + 0x30] %asi, %i4 /* %i4/%i5 = src1 + 0x30 */ + ldda [%l7 + 0x30] %asi, %l0 /* %l0/%l1 = src2 + 0x30 */ + ldda [%i0 + 0x30] %asi, %o2 /* %o2/%o3 = dest + 0x30 */ + prefetch [%i1 + 0x40], #one_read + prefetch [%l7 + 0x40], #one_read + prefetch [%i0 + 0x40], #n_writes + xor %g2, %i2, %g2 + xor %g3, %i3, %g3 + xor %o0, %g2, %o0 + xor %o1, %g3, %o1 + stxa %o0, [%i0 + 0x20] %asi + stxa %o1, [%i0 + 0x28] %asi + xor %l0, %i4, %l0 + xor %l1, %i5, %l1 + xor %o2, %l0, %o2 + xor %o3, %l1, %o3 + stxa %o2, [%i0 + 0x30] %asi + stxa %o3, [%i0 + 0x38] %asi + add %i0, 0x40, %i0 + add %i1, 0x40, %i1 + subcc %g1, 1, %g1 + bne,pt %xcc, 1b + add %l7, 0x40, %l7 + membar #Sync + wr %g7, 0x0, %asi + ret + restore + .size xor_niagara_3, .-xor_niagara_3 + + .globl xor_niagara_4 + .type xor_niagara_4,#function +xor_niagara_4: /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3 */ + save %sp, -192, %sp + prefetch [%i1], #n_writes + prefetch [%i2], #one_read + prefetch [%i3], #one_read + prefetch [%i4], #one_read + rd %asi, %g7 + wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi + srlx %i0, 6, %g1 + mov %i1, %i0 + mov %i2, %i1 + mov %i3, %l7 + mov %i4, %l6 +1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src1 + 0x00 */ + ldda [%l7 + 0x00] %asi, %i4 /* %i4/%i5 = src2 + 0x00 */ + ldda [%l6 + 0x00] %asi, %g2 /* %g2/%g3 = src3 + 0x00 */ + ldda [%i0 + 0x00] %asi, %l0 /* %l0/%l1 = dest + 0x00 */ + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + ldda [%i1 + 0x10] %asi, %i2 /* %i2/%i3 = src1 + 0x10 */ + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + ldda [%l7 + 0x10] %asi, %i4 /* %i4/%i5 = src2 + 0x10 */ + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + stxa %l0, [%i0 + 0x00] %asi + stxa %l1, [%i0 + 0x08] %asi + ldda [%l6 + 0x10] %asi, %g2 /* %g2/%g3 = src3 + 0x10 */ + ldda [%i0 + 0x10] %asi, %l0 /* %l0/%l1 = dest + 0x10 */ + + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + ldda [%i1 + 0x20] %asi, %i2 /* %i2/%i3 = src1 + 0x20 */ + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + ldda [%l7 + 0x20] %asi, %i4 /* %i4/%i5 = src2 + 0x20 */ + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + stxa %l0, [%i0 + 0x10] %asi + stxa %l1, [%i0 + 0x18] %asi + ldda [%l6 + 0x20] %asi, %g2 /* %g2/%g3 = src3 + 0x20 */ + ldda [%i0 + 0x20] %asi, %l0 /* %l0/%l1 = dest + 0x20 */ + + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + ldda [%i1 + 0x30] %asi, %i2 /* %i2/%i3 = src1 + 0x30 */ + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + ldda [%l7 + 0x30] %asi, %i4 /* %i4/%i5 = src2 + 0x30 */ + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + stxa %l0, [%i0 + 0x20] %asi + stxa %l1, [%i0 + 0x28] %asi + ldda [%l6 + 0x30] %asi, %g2 /* %g2/%g3 = src3 + 0x30 */ + ldda [%i0 + 0x30] %asi, %l0 /* %l0/%l1 = dest + 0x30 */ + + prefetch [%i1 + 0x40], #one_read + prefetch [%l7 + 0x40], #one_read + prefetch [%l6 + 0x40], #one_read + prefetch [%i0 + 0x40], #n_writes + + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + stxa %l0, [%i0 + 0x30] %asi + stxa %l1, [%i0 + 0x38] %asi + + add %i0, 0x40, %i0 + add %i1, 0x40, %i1 + add %l7, 0x40, %l7 + subcc %g1, 1, %g1 + bne,pt %xcc, 1b + add %l6, 0x40, %l6 + membar #Sync + wr %g7, 0x0, %asi + ret + restore + .size xor_niagara_4, .-xor_niagara_4 + + .globl xor_niagara_5 + .type xor_niagara_5,#function +xor_niagara_5: /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3, %o5=src4 */ + save %sp, -192, %sp + prefetch [%i1], #n_writes + prefetch [%i2], #one_read + prefetch [%i3], #one_read + prefetch [%i4], #one_read + prefetch [%i5], #one_read + rd %asi, %g7 + wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi + srlx %i0, 6, %g1 + mov %i1, %i0 + mov %i2, %i1 + mov %i3, %l7 + mov %i4, %l6 + mov %i5, %l5 +1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src1 + 0x00 */ + ldda [%l7 + 0x00] %asi, %i4 /* %i4/%i5 = src2 + 0x00 */ + ldda [%l6 + 0x00] %asi, %g2 /* %g2/%g3 = src3 + 0x00 */ + ldda [%l5 + 0x00] %asi, %l0 /* %l0/%l1 = src4 + 0x00 */ + ldda [%i0 + 0x00] %asi, %l2 /* %l2/%l3 = dest + 0x00 */ + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + ldda [%i1 + 0x10] %asi, %i2 /* %i2/%i3 = src1 + 0x10 */ + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + ldda [%l7 + 0x10] %asi, %i4 /* %i4/%i5 = src2 + 0x10 */ + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + ldda [%l6 + 0x10] %asi, %g2 /* %g2/%g3 = src3 + 0x10 */ + xor %l2, %l0, %l2 + xor %l3, %l1, %l3 + stxa %l2, [%i0 + 0x00] %asi + stxa %l3, [%i0 + 0x08] %asi + ldda [%l5 + 0x10] %asi, %l0 /* %l0/%l1 = src4 + 0x10 */ + ldda [%i0 + 0x10] %asi, %l2 /* %l2/%l3 = dest + 0x10 */ + + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + ldda [%i1 + 0x20] %asi, %i2 /* %i2/%i3 = src1 + 0x20 */ + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + ldda [%l7 + 0x20] %asi, %i4 /* %i4/%i5 = src2 + 0x20 */ + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + ldda [%l6 + 0x20] %asi, %g2 /* %g2/%g3 = src3 + 0x20 */ + xor %l2, %l0, %l2 + xor %l3, %l1, %l3 + stxa %l2, [%i0 + 0x10] %asi + stxa %l3, [%i0 + 0x18] %asi + ldda [%l5 + 0x20] %asi, %l0 /* %l0/%l1 = src4 + 0x20 */ + ldda [%i0 + 0x20] %asi, %l2 /* %l2/%l3 = dest + 0x20 */ + + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + ldda [%i1 + 0x30] %asi, %i2 /* %i2/%i3 = src1 + 0x30 */ + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + ldda [%l7 + 0x30] %asi, %i4 /* %i4/%i5 = src2 + 0x30 */ + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + ldda [%l6 + 0x30] %asi, %g2 /* %g2/%g3 = src3 + 0x30 */ + xor %l2, %l0, %l2 + xor %l3, %l1, %l3 + stxa %l2, [%i0 + 0x20] %asi + stxa %l3, [%i0 + 0x28] %asi + ldda [%l5 + 0x30] %asi, %l0 /* %l0/%l1 = src4 + 0x30 */ + ldda [%i0 + 0x30] %asi, %l2 /* %l2/%l3 = dest + 0x30 */ + + prefetch [%i1 + 0x40], #one_read + prefetch [%l7 + 0x40], #one_read + prefetch [%l6 + 0x40], #one_read + prefetch [%l5 + 0x40], #one_read + prefetch [%i0 + 0x40], #n_writes + + xor %i4, %i2, %i4 + xor %i5, %i3, %i5 + xor %g2, %i4, %g2 + xor %g3, %i5, %g3 + xor %l0, %g2, %l0 + xor %l1, %g3, %l1 + xor %l2, %l0, %l2 + xor %l3, %l1, %l3 + stxa %l2, [%i0 + 0x30] %asi + stxa %l3, [%i0 + 0x38] %asi + + add %i0, 0x40, %i0 + add %i1, 0x40, %i1 + add %l7, 0x40, %l7 + add %l6, 0x40, %l6 + subcc %g1, 1, %g1 + bne,pt %xcc, 1b + add %l5, 0x40, %l5 + membar #Sync + wr %g7, 0x0, %asi + ret + restore + .size xor_niagara_5, .-xor_niagara_5 -- cgit v1.2.3 From 8bf68e4d90c4ad862356679ab616726eee1abf93 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Mon, 8 Dec 2008 16:05:49 -0800 Subject: sparc: Kill memcmp_32.S code which has been ifdef'd out for centuries. Signed-off-by: David S. Miller --- arch/sparc/lib/memcmp_32.S | 282 --------------------------------------------- 1 file changed, 282 deletions(-) (limited to 'arch/sparc/lib') diff --git a/arch/sparc/lib/memcmp_32.S b/arch/sparc/lib/memcmp_32.S index cb4bdb0cc2af..9e21f0282659 100644 --- a/arch/sparc/lib/memcmp_32.S +++ b/arch/sparc/lib/memcmp_32.S @@ -3,7 +3,6 @@ .global __memcmp, memcmp __memcmp: memcmp: -#if 1 cmp %o2, 0 ble L3 mov 0, %g3 @@ -29,284 +28,3 @@ L3: retl nop -#else - save %sp, -104, %sp - mov %i2, %o4 - mov %i0, %o0 - - cmp %o4, 15 - ble L72 - mov %i1, %i2 - - andcc %i2, 3, %g0 - be L161 - andcc %o0, 3, %g2 -L75: - ldub [%o0], %g3 - ldub [%i2], %g2 - add %o0,1, %o0 - - subcc %g3, %g2, %i0 - bne L156 - add %i2, 1, %i2 - - andcc %i2, 3, %g0 - bne L75 - add %o4, -1, %o4 - - andcc %o0, 3, %g2 -L161: - bne,a L78 - mov %i2, %i1 - - mov %o0, %i5 - mov %i2, %i3 - srl %o4, 2, %i4 - - cmp %i4, 0 - bge L93 - mov %i4, %g2 - - add %i4, 3, %g2 -L93: - sra %g2, 2, %g2 - sll %g2, 2, %g2 - sub %i4, %g2, %g2 - - cmp %g2, 1 - be,a L88 - add %o0, 4, %i5 - - bg L94 - cmp %g2, 2 - - cmp %g2, 0 - be,a L86 - ld [%o0], %g3 - - b L162 - ld [%i5], %g3 -L94: - be L81 - cmp %g2, 3 - - be,a L83 - add %o0, -4, %i5 - - b L162 - ld [%i5], %g3 -L81: - add %o0, -8, %i5 - ld [%o0], %g3 - add %i2, -8, %i3 - ld [%i2], %g2 - - b L82 - add %i4, 2, %i4 -L83: - ld [%o0], %g4 - add %i2, -4, %i3 - ld [%i2], %g1 - - b L84 - add %i4, 1, %i4 -L86: - b L87 - ld [%i2], %g2 -L88: - add %i2, 4, %i3 - ld [%o0], %g4 - add %i4, -1, %i4 - ld [%i2], %g1 -L95: - ld [%i5], %g3 -L162: - cmp %g4, %g1 - be L87 - ld [%i3], %g2 - - cmp %g4, %g1 -L163: - bleu L114 - mov -1, %i0 - - b L114 - mov 1, %i0 -L87: - ld [%i5 + 4], %g4 - cmp %g3, %g2 - bne L163 - ld [%i3 + 4], %g1 -L84: - ld [%i5 + 8], %g3 - - cmp %g4, %g1 - bne L163 - ld [%i3 + 8], %g2 -L82: - ld [%i5 + 12], %g4 - cmp %g3, %g2 - bne L163 - ld [%i3 + 12], %g1 - - add %i5, 16, %i5 - - addcc %i4, -4, %i4 - bne L95 - add %i3, 16, %i3 - - cmp %g4, %g1 - bne L163 - nop - - b L114 - mov 0, %i0 -L78: - srl %o4, 2, %i0 - and %o0, -4, %i3 - orcc %i0, %g0, %g3 - sll %g2, 3, %o7 - mov 32, %g2 - - bge L129 - sub %g2, %o7, %o1 - - add %i0, 3, %g3 -L129: - sra %g3, 2, %g2 - sll %g2, 2, %g2 - sub %i0, %g2, %g2 - - cmp %g2, 1 - be,a L124 - ld [%i3], %o3 - - bg L130 - cmp %g2, 2 - - cmp %g2, 0 - be,a L122 - ld [%i3], %o2 - - b L164 - sll %o3, %o7, %g3 -L130: - be L117 - cmp %g2, 3 - - be,a L119 - ld [%i3], %g1 - - b L164 - sll %o3, %o7, %g3 -L117: - ld [%i3], %g4 - add %i2, -8, %i1 - ld [%i3 + 4], %o3 - add %i0, 2, %i0 - ld [%i2], %i4 - - b L118 - add %i3, -4, %i3 -L119: - ld [%i3 + 4], %g4 - add %i2, -4, %i1 - ld [%i2], %i5 - - b L120 - add %i0, 1, %i0 -L122: - ld [%i3 + 4], %g1 - ld [%i2], %i4 - - b L123 - add %i3, 4, %i3 -L124: - add %i2, 4, %i1 - ld [%i3 + 4], %o2 - add %i0, -1, %i0 - ld [%i2], %i5 - add %i3, 8, %i3 -L131: - sll %o3, %o7, %g3 -L164: - srl %o2, %o1, %g2 - ld [%i3], %g1 - or %g3, %g2, %g3 - - cmp %g3, %i5 - bne L163 - ld [%i1], %i4 -L123: - sll %o2, %o7, %g3 - srl %g1, %o1, %g2 - ld [%i3 + 4], %g4 - or %g3, %g2, %g3 - - cmp %g3, %i4 - bne L163 - ld [%i1 + 4], %i5 -L120: - sll %g1, %o7, %g3 - srl %g4, %o1, %g2 - ld [%i3 + 8], %o3 - or %g3, %g2, %g3 - - cmp %g3, %i5 - bne L163 - ld [%i1 + 8], %i4 -L118: - sll %g4, %o7, %g3 - srl %o3, %o1, %g2 - ld [%i3 + 12], %o2 - or %g3, %g2, %g3 - - cmp %g3, %i4 - bne L163 - ld [%i1 + 12], %i5 - - add %i3, 16, %i3 - addcc %i0, -4, %i0 - bne L131 - add %i1, 16, %i1 - - sll %o3, %o7, %g3 - srl %o2, %o1, %g2 - or %g3, %g2, %g3 - - cmp %g3, %i5 - be,a L114 - mov 0, %i0 - - b,a L163 -L114: - cmp %i0, 0 - bne L156 - and %o4, -4, %g2 - - add %o0, %g2, %o0 - add %i2, %g2, %i2 - and %o4, 3, %o4 -L72: - cmp %o4, 0 - be L156 - mov 0, %i0 - - ldub [%o0], %g3 -L165: - ldub [%i2], %g2 - add %o0, 1, %o0 - - subcc %g3, %g2, %i0 - bne L156 - add %i2, 1, %i2 - - addcc %o4, -1, %o4 - bne,a L165 - ldub [%o0], %g3 - - mov 0, %i0 -L156: - ret - restore -#endif -- cgit v1.2.3 From ae984d72e0632782dd98c3fcf469b9045ad0d449 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 9 Dec 2008 01:07:09 -0800 Subject: sparc: Unify strlen assembler. Use the new asm/asm.h header to help commonize the strlen assembler between 32-bit and 64-bit While we're here, use proper linux/linkage.h macros instead of by-hand stuff. Signed-off-by: David S. Miller --- arch/sparc/lib/Makefile | 2 +- arch/sparc/lib/strlen.S | 80 +++++++++++++++++++++++++++++++++++++++++++++ arch/sparc/lib/strlen_32.S | 81 ---------------------------------------------- arch/sparc/lib/strlen_64.S | 80 --------------------------------------------- 4 files changed, 81 insertions(+), 162 deletions(-) create mode 100644 arch/sparc/lib/strlen.S delete mode 100644 arch/sparc/lib/strlen_32.S delete mode 100644 arch/sparc/lib/strlen_64.S (limited to 'arch/sparc/lib') diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile index 05ae5c945e35..0db2c61a0f78 100644 --- a/arch/sparc/lib/Makefile +++ b/arch/sparc/lib/Makefile @@ -6,7 +6,7 @@ ccflags-y := -Werror lib-$(CONFIG_SPARC32) += mul.o rem.o sdiv.o udiv.o umul.o urem.o ashrdi3.o lib-$(CONFIG_SPARC32) += memcpy.o memset.o -lib-y += strlen_$(BITS).o +lib-y += strlen.o lib-y += checksum_$(BITS).o lib-$(CONFIG_SPARC32) += blockops.o lib-y += memscan_$(BITS).o memcmp_$(BITS).o strncmp_$(BITS).o diff --git a/arch/sparc/lib/strlen.S b/arch/sparc/lib/strlen.S new file mode 100644 index 000000000000..536f83507fbf --- /dev/null +++ b/arch/sparc/lib/strlen.S @@ -0,0 +1,80 @@ +/* strlen.S: Sparc optimized strlen code + * Hand optimized from GNU libc's strlen + * Copyright (C) 1991,1996 Free Software Foundation + * Copyright (C) 1996,2008 David S. Miller (davem@davemloft.net) + * Copyright (C) 1996, 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include + +#define LO_MAGIC 0x01010101 +#define HI_MAGIC 0x80808080 + + .text +ENTRY(strlen) + mov %o0, %o1 + andcc %o0, 3, %g0 + BRANCH32(be, pt, 9f) + sethi %hi(HI_MAGIC), %o4 + ldub [%o0], %o5 + BRANCH_REG_ZERO(pn, %o5, 11f) + add %o0, 1, %o0 + andcc %o0, 3, %g0 + BRANCH32(be, pn, 4f) + or %o4, %lo(HI_MAGIC), %o3 + ldub [%o0], %o5 + BRANCH_REG_ZERO(pn, %o5, 12f) + add %o0, 1, %o0 + andcc %o0, 3, %g0 + BRANCH32(be, pt, 5f) + sethi %hi(LO_MAGIC), %o4 + ldub [%o0], %o5 + BRANCH_REG_ZERO(pn, %o5, 13f) + add %o0, 1, %o0 + BRANCH32(ba, pt, 8f) + or %o4, %lo(LO_MAGIC), %o2 +9: + or %o4, %lo(HI_MAGIC), %o3 +4: + sethi %hi(LO_MAGIC), %o4 +5: + or %o4, %lo(LO_MAGIC), %o2 +8: + ld [%o0], %o5 +2: + sub %o5, %o2, %o4 + andcc %o4, %o3, %g0 + BRANCH32(be, pt, 8b) + add %o0, 4, %o0 + + /* Check every byte. */ + srl %o5, 24, %g7 + andcc %g7, 0xff, %g0 + BRANCH32(be, pn, 1f) + add %o0, -4, %o4 + srl %o5, 16, %g7 + andcc %g7, 0xff, %g0 + BRANCH32(be, pn, 1f) + add %o4, 1, %o4 + srl %o5, 8, %g7 + andcc %g7, 0xff, %g0 + BRANCH32(be, pn, 1f) + add %o4, 1, %o4 + andcc %o5, 0xff, %g0 + BRANCH32_ANNUL(bne, pt, 2b) + ld [%o0], %o5 + add %o4, 1, %o4 +1: + retl + sub %o4, %o1, %o0 +11: + retl + mov 0, %o0 +12: + retl + mov 1, %o0 +13: + retl + mov 2, %o0 +ENDPROC(strlen) diff --git a/arch/sparc/lib/strlen_32.S b/arch/sparc/lib/strlen_32.S deleted file mode 100644 index ed9a763368cd..000000000000 --- a/arch/sparc/lib/strlen_32.S +++ /dev/null @@ -1,81 +0,0 @@ -/* strlen.S: Sparc optimized strlen code - * Hand optimized from GNU libc's strlen - * Copyright (C) 1991,1996 Free Software Foundation - * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) - * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#define LO_MAGIC 0x01010101 -#define HI_MAGIC 0x80808080 - -0: - ldub [%o0], %o5 - cmp %o5, 0 - be 1f - add %o0, 1, %o0 - andcc %o0, 3, %g0 - be 4f - or %o4, %lo(HI_MAGIC), %o3 - ldub [%o0], %o5 - cmp %o5, 0 - be 2f - add %o0, 1, %o0 - andcc %o0, 3, %g0 - be 5f - sethi %hi(LO_MAGIC), %o4 - ldub [%o0], %o5 - cmp %o5, 0 - be 3f - add %o0, 1, %o0 - b 8f - or %o4, %lo(LO_MAGIC), %o2 -1: - retl - mov 0, %o0 -2: - retl - mov 1, %o0 -3: - retl - mov 2, %o0 - - .align 4 - .global strlen -strlen: - mov %o0, %o1 - andcc %o0, 3, %g0 - bne 0b - sethi %hi(HI_MAGIC), %o4 - or %o4, %lo(HI_MAGIC), %o3 -4: - sethi %hi(LO_MAGIC), %o4 -5: - or %o4, %lo(LO_MAGIC), %o2 -8: - ld [%o0], %o5 -2: - sub %o5, %o2, %o4 - andcc %o4, %o3, %g0 - be 8b - add %o0, 4, %o0 - - /* Check every byte. */ - srl %o5, 24, %g5 - andcc %g5, 0xff, %g0 - be 1f - add %o0, -4, %o4 - srl %o5, 16, %g5 - andcc %g5, 0xff, %g0 - be 1f - add %o4, 1, %o4 - srl %o5, 8, %g5 - andcc %g5, 0xff, %g0 - be 1f - add %o4, 1, %o4 - andcc %o5, 0xff, %g0 - bne,a 2b - ld [%o0], %o5 - add %o4, 1, %o4 -1: - retl - sub %o4, %o1, %o0 diff --git a/arch/sparc/lib/strlen_64.S b/arch/sparc/lib/strlen_64.S deleted file mode 100644 index e9ba1920d818..000000000000 --- a/arch/sparc/lib/strlen_64.S +++ /dev/null @@ -1,80 +0,0 @@ -/* strlen.S: Sparc64 optimized strlen code - * Hand optimized from GNU libc's strlen - * Copyright (C) 1991,1996 Free Software Foundation - * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) - * Copyright (C) 1996, 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#define LO_MAGIC 0x01010101 -#define HI_MAGIC 0x80808080 - - .align 32 - .globl strlen - .type strlen,#function -strlen: - mov %o0, %o1 - andcc %o0, 3, %g0 - be,pt %icc, 9f - sethi %hi(HI_MAGIC), %o4 - ldub [%o0], %o5 - brz,pn %o5, 11f - add %o0, 1, %o0 - andcc %o0, 3, %g0 - be,pn %icc, 4f - or %o4, %lo(HI_MAGIC), %o3 - ldub [%o0], %o5 - brz,pn %o5, 12f - add %o0, 1, %o0 - andcc %o0, 3, %g0 - be,pt %icc, 5f - sethi %hi(LO_MAGIC), %o4 - ldub [%o0], %o5 - brz,pn %o5, 13f - add %o0, 1, %o0 - ba,pt %icc, 8f - or %o4, %lo(LO_MAGIC), %o2 -9: - or %o4, %lo(HI_MAGIC), %o3 -4: - sethi %hi(LO_MAGIC), %o4 -5: - or %o4, %lo(LO_MAGIC), %o2 -8: - ld [%o0], %o5 -2: - sub %o5, %o2, %o4 - andcc %o4, %o3, %g0 - be,pt %icc, 8b - add %o0, 4, %o0 - - /* Check every byte. */ - srl %o5, 24, %g7 - andcc %g7, 0xff, %g0 - be,pn %icc, 1f - add %o0, -4, %o4 - srl %o5, 16, %g7 - andcc %g7, 0xff, %g0 - be,pn %icc, 1f - add %o4, 1, %o4 - srl %o5, 8, %g7 - andcc %g7, 0xff, %g0 - be,pn %icc, 1f - add %o4, 1, %o4 - andcc %o5, 0xff, %g0 - bne,a,pt %icc, 2b - ld [%o0], %o5 - add %o4, 1, %o4 -1: - retl - sub %o4, %o1, %o0 -11: - retl - mov 0, %o0 -12: - retl - mov 1, %o0 -13: - retl - mov 2, %o0 - - .size strlen, .-strlen -- cgit v1.2.3 From 18cdae68e7bb24f33883e58f366cde38ea89ba17 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Tue, 9 Dec 2008 04:09:07 -0800 Subject: sparc: Commonize memcmp assembler. Signed-off-by: David S. Miller --- arch/sparc/lib/Makefile | 2 +- arch/sparc/lib/memcmp.S | 27 +++++++++++++++++++++++++++ arch/sparc/lib/memcmp_32.S | 30 ------------------------------ arch/sparc/lib/memcmp_64.S | 28 ---------------------------- 4 files changed, 28 insertions(+), 59 deletions(-) create mode 100644 arch/sparc/lib/memcmp.S delete mode 100644 arch/sparc/lib/memcmp_32.S delete mode 100644 arch/sparc/lib/memcmp_64.S (limited to 'arch/sparc/lib') diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile index 0db2c61a0f78..375016e19144 100644 --- a/arch/sparc/lib/Makefile +++ b/arch/sparc/lib/Makefile @@ -9,7 +9,7 @@ lib-$(CONFIG_SPARC32) += memcpy.o memset.o lib-y += strlen.o lib-y += checksum_$(BITS).o lib-$(CONFIG_SPARC32) += blockops.o -lib-y += memscan_$(BITS).o memcmp_$(BITS).o strncmp_$(BITS).o +lib-y += memscan_$(BITS).o memcmp.o strncmp_$(BITS).o lib-y += strncpy_from_user_$(BITS).o strlen_user_$(BITS).o lib-$(CONFIG_SPARC32) += divdi3.o udivdi3.o lib-$(CONFIG_SPARC32) += copy_user.o locks.o diff --git a/arch/sparc/lib/memcmp.S b/arch/sparc/lib/memcmp.S new file mode 100644 index 000000000000..efa106c41ed0 --- /dev/null +++ b/arch/sparc/lib/memcmp.S @@ -0,0 +1,27 @@ +/* Sparc optimized memcmp code. + * + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Copyright (C) 2000, 2008 David S. Miller (davem@davemloft.net) + */ + +#include +#include + + .text +ENTRY(memcmp) + cmp %o2, 0 +1: BRANCH32(be, pn, 2f) + nop + ldub [%o0], %g7 + ldub [%o1], %g3 + sub %o2, 1, %o2 + add %o0, 1, %o0 + add %o1, 1, %o1 + subcc %g7, %g3, %g3 + BRANCH32(be, pt, 1b) + cmp %o2, 0 + retl + mov %g3, %o0 +2: retl + mov 0, %o0 +ENDPROC(memcmp) diff --git a/arch/sparc/lib/memcmp_32.S b/arch/sparc/lib/memcmp_32.S deleted file mode 100644 index 9e21f0282659..000000000000 --- a/arch/sparc/lib/memcmp_32.S +++ /dev/null @@ -1,30 +0,0 @@ - .text - .align 4 - .global __memcmp, memcmp -__memcmp: -memcmp: - cmp %o2, 0 - ble L3 - mov 0, %g3 -L5: - ldub [%o0], %g2 - ldub [%o1], %g3 - sub %g2, %g3, %g2 - mov %g2, %g3 - sll %g2, 24, %g2 - - cmp %g2, 0 - bne L3 - add %o0, 1, %o0 - - add %o2, -1, %o2 - - cmp %o2, 0 - bg L5 - add %o1, 1, %o1 -L3: - sll %g3, 24, %o0 - sra %o0, 24, %o0 - - retl - nop diff --git a/arch/sparc/lib/memcmp_64.S b/arch/sparc/lib/memcmp_64.S deleted file mode 100644 index d3fdaa898566..000000000000 --- a/arch/sparc/lib/memcmp_64.S +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Sparc64 optimized memcmp code. - * - * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - * Copyright (C) 2000 David S. Miller (davem@redhat.com) - */ - - .text - .align 32 - .globl __memcmp, memcmp -__memcmp: -memcmp: - cmp %o2, 0 ! IEU1 Group -loop: be,pn %icc, ret_0 ! CTI - nop ! IEU0 - ldub [%o0], %g7 ! LSU Group - ldub [%o1], %g3 ! LSU Group - sub %o2, 1, %o2 ! IEU0 - add %o0, 1, %o0 ! IEU1 - add %o1, 1, %o1 ! IEU0 Group - subcc %g7, %g3, %g3 ! IEU1 Group - be,pt %icc, loop ! CTI - cmp %o2, 0 ! IEU1 Group - -ret_n0: retl - mov %g3, %o0 -ret_0: retl - mov 0, %o0 -- cgit v1.2.3