From b5f42db0a782d409e5fe0efb6e1387e45774c70b Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Thu, 8 Nov 2007 18:38:14 +0900 Subject: sh: Move CPU subtypes to Kconfig.sh. We intend to share the mm options, so move the SH-only subtypes up a level. Signed-off-by: Paul Mundt --- arch/sh/mm/Kconfig | 190 ----------------------------------------------------- 1 file changed, 190 deletions(-) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/Kconfig b/arch/sh/mm/Kconfig index 1265f204f7d1..998c5c86346f 100644 --- a/arch/sh/mm/Kconfig +++ b/arch/sh/mm/Kconfig @@ -1,193 +1,3 @@ -# -# Processor families -# -config CPU_SH2 - bool - -config CPU_SH2A - bool - select CPU_SH2 - -config CPU_SH3 - bool - select CPU_HAS_INTEVT - select CPU_HAS_SR_RB - -config CPU_SH4 - bool - select CPU_HAS_INTEVT - select CPU_HAS_SR_RB - select CPU_HAS_PTEA if !CPU_SH4A || CPU_SHX2 - select CPU_HAS_FPU if !CPU_SH4AL_DSP - -config CPU_SH4A - bool - select CPU_SH4 - -config CPU_SH4AL_DSP - bool - select CPU_SH4A - select CPU_HAS_DSP - -config CPU_SHX2 - bool - -config CPU_SHX3 - bool - -choice - prompt "Processor sub-type selection" - -# -# Processor subtypes -# - -# SH-2 Processor Support - -config CPU_SUBTYPE_SH7619 - bool "Support SH7619 processor" - select CPU_SH2 - -# SH-2A Processor Support - -config CPU_SUBTYPE_SH7206 - bool "Support SH7206 processor" - select CPU_SH2A - -# SH-3 Processor Support - -config CPU_SUBTYPE_SH7705 - bool "Support SH7705 processor" - select CPU_SH3 - -config CPU_SUBTYPE_SH7706 - bool "Support SH7706 processor" - select CPU_SH3 - help - Select SH7706 if you have a 133 Mhz SH-3 HD6417706 CPU. - -config CPU_SUBTYPE_SH7707 - bool "Support SH7707 processor" - select CPU_SH3 - help - Select SH7707 if you have a 60 Mhz SH-3 HD6417707 CPU. - -config CPU_SUBTYPE_SH7708 - bool "Support SH7708 processor" - select CPU_SH3 - help - Select SH7708 if you have a 60 Mhz SH-3 HD6417708S or - if you have a 100 Mhz SH-3 HD6417708R CPU. - -config CPU_SUBTYPE_SH7709 - bool "Support SH7709 processor" - select CPU_SH3 - help - Select SH7709 if you have a 80 Mhz SH-3 HD6417709 CPU. - -config CPU_SUBTYPE_SH7710 - bool "Support SH7710 processor" - select CPU_SH3 - select CPU_HAS_DSP - help - Select SH7710 if you have a SH3-DSP SH7710 CPU. - -config CPU_SUBTYPE_SH7712 - bool "Support SH7712 processor" - select CPU_SH3 - select CPU_HAS_DSP - help - Select SH7712 if you have a SH3-DSP SH7712 CPU. - -config CPU_SUBTYPE_SH7720 - bool "Support SH7720 processor" - select CPU_SH3 - select CPU_HAS_DSP - help - Select SH7720 if you have a SH3-DSP SH7720 CPU. - -# SH-4 Processor Support - -config CPU_SUBTYPE_SH7750 - bool "Support SH7750 processor" - select CPU_SH4 - help - Select SH7750 if you have a 200 Mhz SH-4 HD6417750 CPU. - -config CPU_SUBTYPE_SH7091 - bool "Support SH7091 processor" - select CPU_SH4 - help - Select SH7091 if you have an SH-4 based Sega device (such as - the Dreamcast, Naomi, and Naomi 2). - -config CPU_SUBTYPE_SH7750R - bool "Support SH7750R processor" - select CPU_SH4 - -config CPU_SUBTYPE_SH7750S - bool "Support SH7750S processor" - select CPU_SH4 - -config CPU_SUBTYPE_SH7751 - bool "Support SH7751 processor" - select CPU_SH4 - help - Select SH7751 if you have a 166 Mhz SH-4 HD6417751 CPU, - or if you have a HD6417751R CPU. - -config CPU_SUBTYPE_SH7751R - bool "Support SH7751R processor" - select CPU_SH4 - -config CPU_SUBTYPE_SH7760 - bool "Support SH7760 processor" - select CPU_SH4 - -config CPU_SUBTYPE_SH4_202 - bool "Support SH4-202 processor" - select CPU_SH4 - -# SH-4A Processor Support - -config CPU_SUBTYPE_SH7770 - bool "Support SH7770 processor" - select CPU_SH4A - -config CPU_SUBTYPE_SH7780 - bool "Support SH7780 processor" - select CPU_SH4A - -config CPU_SUBTYPE_SH7785 - bool "Support SH7785 processor" - select CPU_SH4A - select CPU_SHX2 - select ARCH_SPARSEMEM_ENABLE - select SYS_SUPPORTS_NUMA - -config CPU_SUBTYPE_SHX3 - bool "Support SH-X3 processor" - select CPU_SH4A - select CPU_SHX3 - select ARCH_SPARSEMEM_ENABLE - select SYS_SUPPORTS_NUMA - select SYS_SUPPORTS_SMP - -# SH4AL-DSP Processor Support - -config CPU_SUBTYPE_SH7343 - bool "Support SH7343 processor" - select CPU_SH4AL_DSP - -config CPU_SUBTYPE_SH7722 - bool "Support SH7722 processor" - select CPU_SH4AL_DSP - select CPU_SHX2 - select ARCH_SPARSEMEM_ENABLE - select SYS_SUPPORTS_NUMA - -endchoice - menu "Memory management options" config QUICKLIST -- cgit v1.2.3 From a09063da33a222adaceea85c9d168cbbb3e626b0 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Thu, 8 Nov 2007 18:54:16 +0900 Subject: sh: Switch Kconfig.sh64 to use arch/sh/mm/Kconfig. Consolidates the HUGETLB definitions and others. Signed-off-by: Paul Mundt --- arch/sh/mm/Kconfig | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/Kconfig b/arch/sh/mm/Kconfig index 998c5c86346f..a7bbc2cbbf8c 100644 --- a/arch/sh/mm/Kconfig +++ b/arch/sh/mm/Kconfig @@ -66,7 +66,7 @@ config X2TLB config VSYSCALL bool "Support vsyscall page" - depends on MMU + depends on MMU && (CPU_SH3 || CPU_SH4) default y help This will enable support for the kernel mapping a vDSO page @@ -175,6 +175,10 @@ config HUGETLB_PAGE_SIZE_64MB bool "64MB" depends on X2TLB +config HUGETLB_PAGE_SIZE_512MB + bool "512MB" + depends on CPU_SH5 + endchoice source "mm/Kconfig" @@ -202,12 +206,12 @@ config SH_DIRECT_MAPPED choice prompt "Cache mode" - default CACHE_WRITEBACK if CPU_SH2A || CPU_SH3 || CPU_SH4 + default CACHE_WRITEBACK if CPU_SH2A || CPU_SH3 || CPU_SH4 || CPU_SH5 default CACHE_WRITETHROUGH if (CPU_SH2 && !CPU_SH2A) config CACHE_WRITEBACK bool "Write-back" - depends on CPU_SH2A || CPU_SH3 || CPU_SH4 + depends on CPU_SH2A || CPU_SH3 || CPU_SH4 || CPU_SH5 config CACHE_WRITETHROUGH bool "Write-through" -- cgit v1.2.3 From 36bcd39dbca824daffe16d607ae574b6edc7d31a Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Sat, 10 Nov 2007 19:16:55 +0900 Subject: sh: Split out 29-bit and 32-bit physical mode definitions. Signed-off-by: Paul Mundt --- arch/sh/mm/Kconfig | 11 +++++++++++ arch/sh/mm/Makefile | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/Kconfig b/arch/sh/mm/Kconfig index a7bbc2cbbf8c..8192c872a69a 100644 --- a/arch/sh/mm/Kconfig +++ b/arch/sh/mm/Kconfig @@ -46,9 +46,20 @@ config MEMORY_SIZE as 0x00400000 which was the default value before this became configurable. +# Physical addressing modes + +config 29BIT + def_bool !32BIT + depends on SUPERH32 + config 32BIT + bool + default y if CPU_SH5 + +config PMB bool "Support 32-bit physical addressing through PMB" depends on MMU && (CPU_SUBTYPE_SH7780 || CPU_SUBTYPE_SH7785) + select 32BIT default y help If you say Y here, physical addressing will be extended to diff --git a/arch/sh/mm/Makefile b/arch/sh/mm/Makefile index aa44607f072d..d35caccd0aaa 100644 --- a/arch/sh/mm/Makefile +++ b/arch/sh/mm/Makefile @@ -31,7 +31,7 @@ endif endif obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o -obj-$(CONFIG_32BIT) += pmb.o +obj-$(CONFIG_PMB) += pmb.o obj-$(CONFIG_NUMA) += numa.o EXTRA_CFLAGS += -Werror -- cgit v1.2.3 From 8a7bcf0dd0d49fe8b0071adef0dfe8610abdffaa Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Sun, 11 Nov 2007 17:07:06 +0900 Subject: sh: Add SH-5 support to the consistent DMA impl. Signed-off-by: Paul Mundt --- arch/sh/mm/consistent.c | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/consistent.c b/arch/sh/mm/consistent.c index e220c29a3c00..65ad30031ad7 100644 --- a/arch/sh/mm/consistent.c +++ b/arch/sh/mm/consistent.c @@ -1,7 +1,7 @@ /* * arch/sh/mm/consistent.c * - * Copyright (C) 2004 Paul Mundt + * Copyright (C) 2004 - 2007 Paul Mundt * * This file is subject to the terms and conditions of the GNU General Public * License. See the file "COPYING" in the main directory of this archive @@ -16,7 +16,7 @@ void *consistent_alloc(gfp_t gfp, size_t size, dma_addr_t *handle) { struct page *page, *end, *free; - void *ret; + void *ret, *vp; int order; size = PAGE_ALIGN(size); @@ -28,13 +28,20 @@ void *consistent_alloc(gfp_t gfp, size_t size, dma_addr_t *handle) split_page(page, order); ret = page_address(page); - memset(ret, 0, size); *handle = virt_to_phys(ret); + vp = ioremap_nocache(*handle, size); + if (!vp) { + free_pages((unsigned long)ret, order); + return NULL; + } + + memset(vp, 0, size); + /* * We must flush the cache before we pass it on to the device */ - __flush_purge_region(ret, size); + dma_cache_sync(NULL, ret, size, DMA_BIDIRECTIONAL); page = virt_to_page(ret); free = page + (size >> PAGE_SHIFT); @@ -47,24 +54,31 @@ void *consistent_alloc(gfp_t gfp, size_t size, dma_addr_t *handle) } } - return P2SEGADDR(ret); + return vp; } +EXPORT_SYMBOL(consistent_alloc); -void consistent_free(void *vaddr, size_t size) +void consistent_free(void *vaddr, size_t size, dma_addr_t dma_handle) { - unsigned long addr = P1SEGADDR((unsigned long)vaddr); - struct page *page=virt_to_page(addr); - int num_pages=(size+PAGE_SIZE-1) >> PAGE_SHIFT; - int i; + struct page *page; + unsigned long addr; - for(i=0;i Date: Sun, 11 Nov 2007 17:24:50 +0900 Subject: sh: Split out extable.c _32 and _64 variants. Signed-off-by: Paul Mundt --- arch/sh/mm/Makefile | 40 +++---------------------- arch/sh/mm/Makefile_32 | 37 +++++++++++++++++++++++ arch/sh/mm/Makefile_64 | 11 +++++++ arch/sh/mm/extable.c | 21 ------------- arch/sh/mm/extable_32.c | 21 +++++++++++++ arch/sh/mm/extable_64.c | 80 +++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 153 insertions(+), 57 deletions(-) create mode 100644 arch/sh/mm/Makefile_32 create mode 100644 arch/sh/mm/Makefile_64 delete mode 100644 arch/sh/mm/extable.c create mode 100644 arch/sh/mm/extable_32.c create mode 100644 arch/sh/mm/extable_64.c (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/Makefile b/arch/sh/mm/Makefile index d35caccd0aaa..9f4bc3d90b1e 100644 --- a/arch/sh/mm/Makefile +++ b/arch/sh/mm/Makefile @@ -1,37 +1,5 @@ -# -# Makefile for the Linux SuperH-specific parts of the memory manager. -# - -obj-y := init.o extable.o consistent.o - -ifndef CONFIG_CACHE_OFF -obj-$(CONFIG_CPU_SH2) += cache-sh2.o -obj-$(CONFIG_CPU_SH3) += cache-sh3.o -obj-$(CONFIG_CPU_SH4) += cache-sh4.o -obj-$(CONFIG_SH7705_CACHE_32KB) += cache-sh7705.o +ifeq ($(CONFIG_SUPERH32),y) +include ${srctree}/arch/sh/mm/Makefile_32 +else +include ${srctree}/arch/sh/mm/Makefile_64 endif - -mmu-y := tlb-nommu.o pg-nommu.o -mmu-$(CONFIG_MMU) := fault.o clear_page.o copy_page.o tlb-flush.o \ - ioremap.o - -obj-y += $(mmu-y) - -ifdef CONFIG_DEBUG_FS -obj-$(CONFIG_CPU_SH4) += cache-debugfs.o -endif - -ifdef CONFIG_MMU -obj-$(CONFIG_CPU_SH3) += tlb-sh3.o -obj-$(CONFIG_CPU_SH4) += tlb-sh4.o -ifndef CONFIG_CACHE_OFF -obj-$(CONFIG_CPU_SH4) += pg-sh4.o -obj-$(CONFIG_SH7705_CACHE_32KB) += pg-sh7705.o -endif -endif - -obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o -obj-$(CONFIG_PMB) += pmb.o -obj-$(CONFIG_NUMA) += numa.o - -EXTRA_CFLAGS += -Werror diff --git a/arch/sh/mm/Makefile_32 b/arch/sh/mm/Makefile_32 new file mode 100644 index 000000000000..cc68b265c025 --- /dev/null +++ b/arch/sh/mm/Makefile_32 @@ -0,0 +1,37 @@ +# +# Makefile for the Linux SuperH-specific parts of the memory manager. +# + +obj-y := init.o extable_32.o consistent.o + +ifndef CONFIG_CACHE_OFF +obj-$(CONFIG_CPU_SH2) += cache-sh2.o +obj-$(CONFIG_CPU_SH3) += cache-sh3.o +obj-$(CONFIG_CPU_SH4) += cache-sh4.o +obj-$(CONFIG_SH7705_CACHE_32KB) += cache-sh7705.o +endif + +mmu-y := tlb-nommu.o pg-nommu.o +mmu-$(CONFIG_MMU) := fault.o clear_page.o copy_page.o tlb-flush.o \ + ioremap.o + +obj-y += $(mmu-y) + +ifdef CONFIG_DEBUG_FS +obj-$(CONFIG_CPU_SH4) += cache-debugfs.o +endif + +ifdef CONFIG_MMU +obj-$(CONFIG_CPU_SH3) += tlb-sh3.o +obj-$(CONFIG_CPU_SH4) += tlb-sh4.o +ifndef CONFIG_CACHE_OFF +obj-$(CONFIG_CPU_SH4) += pg-sh4.o +obj-$(CONFIG_SH7705_CACHE_32KB) += pg-sh7705.o +endif +endif + +obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o +obj-$(CONFIG_PMB) += pmb.o +obj-$(CONFIG_NUMA) += numa.o + +EXTRA_CFLAGS += -Werror diff --git a/arch/sh/mm/Makefile_64 b/arch/sh/mm/Makefile_64 new file mode 100644 index 000000000000..de964a1500f4 --- /dev/null +++ b/arch/sh/mm/Makefile_64 @@ -0,0 +1,11 @@ +# +# Makefile for the Linux SuperH-specific parts of the memory manager. +# + +obj-y := init.o extable_64.o consistent.o + +obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o +obj-$(CONFIG_PMB) += pmb.o +obj-$(CONFIG_NUMA) += numa.o + +EXTRA_CFLAGS += -Werror diff --git a/arch/sh/mm/extable.c b/arch/sh/mm/extable.c deleted file mode 100644 index c1cf4463d09d..000000000000 --- a/arch/sh/mm/extable.c +++ /dev/null @@ -1,21 +0,0 @@ -/* - * linux/arch/sh/mm/extable.c - * Taken from: - * linux/arch/i386/mm/extable.c - */ - -#include -#include - -int fixup_exception(struct pt_regs *regs) -{ - const struct exception_table_entry *fixup; - - fixup = search_exception_tables(regs->pc); - if (fixup) { - regs->pc = fixup->fixup; - return 1; - } - - return 0; -} diff --git a/arch/sh/mm/extable_32.c b/arch/sh/mm/extable_32.c new file mode 100644 index 000000000000..c1cf4463d09d --- /dev/null +++ b/arch/sh/mm/extable_32.c @@ -0,0 +1,21 @@ +/* + * linux/arch/sh/mm/extable.c + * Taken from: + * linux/arch/i386/mm/extable.c + */ + +#include +#include + +int fixup_exception(struct pt_regs *regs) +{ + const struct exception_table_entry *fixup; + + fixup = search_exception_tables(regs->pc); + if (fixup) { + regs->pc = fixup->fixup; + return 1; + } + + return 0; +} diff --git a/arch/sh/mm/extable_64.c b/arch/sh/mm/extable_64.c new file mode 100644 index 000000000000..a2e6e0563772 --- /dev/null +++ b/arch/sh/mm/extable_64.c @@ -0,0 +1,80 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * arch/sh64/mm/extable.c + * + * Copyright (C) 2003 Richard Curnow + * Copyright (C) 2003, 2004 Paul Mundt + * + * Cloned from the 2.5 SH version.. + */ +#include +#include +#include + +extern unsigned long copy_user_memcpy, copy_user_memcpy_end; +extern void __copy_user_fixup(void); + +static const struct exception_table_entry __copy_user_fixup_ex = { + .fixup = (unsigned long)&__copy_user_fixup, +}; + +/* Some functions that may trap due to a bad user-mode address have too many loads + and stores in them to make it at all practical to label each one and put them all in + the main exception table. + + In particular, the fast memcpy routine is like this. It's fix-up is just to fall back + to a slow byte-at-a-time copy, which is handled the conventional way. So it's functionally + OK to just handle any trap occurring in the fast memcpy with that fixup. */ +static const struct exception_table_entry *check_exception_ranges(unsigned long addr) +{ + if ((addr >= (unsigned long)©_user_memcpy) && + (addr <= (unsigned long)©_user_memcpy_end)) + return &__copy_user_fixup_ex; + + return NULL; +} + +/* Simple binary search */ +const struct exception_table_entry * +search_extable(const struct exception_table_entry *first, + const struct exception_table_entry *last, + unsigned long value) +{ + const struct exception_table_entry *mid; + + mid = check_exception_ranges(value); + if (mid) + return mid; + + while (first <= last) { + long diff; + + mid = (last - first) / 2 + first; + diff = mid->insn - value; + if (diff == 0) + return mid; + else if (diff < 0) + first = mid+1; + else + last = mid-1; + } + + return NULL; +} + +int fixup_exception(struct pt_regs *regs) +{ + const struct exception_table_entry *fixup; + + fixup = search_exception_tables(regs->pc); + if (fixup) { + regs->pc = fixup->fixup; + return 1; + } + + return 0; +} + -- cgit v1.2.3 From 079060c6ff6edd32955c3e511024578db89a717b Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Sun, 11 Nov 2007 17:25:10 +0900 Subject: sh: Fix up 64kB page size selection on SH-5. Signed-off-by: Paul Mundt --- arch/sh/mm/Kconfig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/Kconfig b/arch/sh/mm/Kconfig index 8192c872a69a..7ff26c1f4e9c 100644 --- a/arch/sh/mm/Kconfig +++ b/arch/sh/mm/Kconfig @@ -156,7 +156,7 @@ config PAGE_SIZE_8KB config PAGE_SIZE_64KB bool "64kB" - depends on CPU_SH4 + depends on CPU_SH4 || CPU_SH5 help This enables support for 64kB pages, possible on all SH-4 CPUs and later. @@ -165,7 +165,7 @@ endchoice choice prompt "HugeTLB page size" - depends on HUGETLB_PAGE && CPU_SH4 && MMU + depends on HUGETLB_PAGE && (CPU_SH4 || CPU_SH5) && MMU default HUGETLB_PAGE_SIZE_64K config HUGETLB_PAGE_SIZE_64K -- cgit v1.2.3 From ad81eb91731fe7e43ce5c91278bc33bded6cbda1 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Sun, 11 Nov 2007 18:43:33 +0900 Subject: sh: Split out ioremap in to _32 and _64 variants. Signed-off-by: Paul Mundt --- arch/sh/mm/Makefile_32 | 2 +- arch/sh/mm/Makefile_64 | 5 + arch/sh/mm/ioremap.c | 150 ------------------- arch/sh/mm/ioremap_32.c | 150 +++++++++++++++++++ arch/sh/mm/ioremap_64.c | 387 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 543 insertions(+), 151 deletions(-) delete mode 100644 arch/sh/mm/ioremap.c create mode 100644 arch/sh/mm/ioremap_32.c create mode 100644 arch/sh/mm/ioremap_64.c (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/Makefile_32 b/arch/sh/mm/Makefile_32 index cc68b265c025..b43884269c2a 100644 --- a/arch/sh/mm/Makefile_32 +++ b/arch/sh/mm/Makefile_32 @@ -13,7 +13,7 @@ endif mmu-y := tlb-nommu.o pg-nommu.o mmu-$(CONFIG_MMU) := fault.o clear_page.o copy_page.o tlb-flush.o \ - ioremap.o + ioremap_32.o obj-y += $(mmu-y) diff --git a/arch/sh/mm/Makefile_64 b/arch/sh/mm/Makefile_64 index de964a1500f4..177f92863e4e 100644 --- a/arch/sh/mm/Makefile_64 +++ b/arch/sh/mm/Makefile_64 @@ -4,6 +4,11 @@ obj-y := init.o extable_64.o consistent.o +mmu-y := tlb-nommu.o pg-nommu.o +mmu-$(CONFIG_MMU) := ioremap_64.o + +obj-y += $(mmu-y) + obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_PMB) += pmb.o obj-$(CONFIG_NUMA) += numa.o diff --git a/arch/sh/mm/ioremap.c b/arch/sh/mm/ioremap.c deleted file mode 100644 index 0c7b7e33abdc..000000000000 --- a/arch/sh/mm/ioremap.c +++ /dev/null @@ -1,150 +0,0 @@ -/* - * arch/sh/mm/ioremap.c - * - * Re-map IO memory to kernel address space so that we can access it. - * This is needed for high PCI addresses that aren't mapped in the - * 640k-1MB IO memory area on PC's - * - * (C) Copyright 1995 1996 Linus Torvalds - * (C) Copyright 2005, 2006 Paul Mundt - * - * This file is subject to the terms and conditions of the GNU General - * Public License. See the file "COPYING" in the main directory of this - * archive for more details. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * Remap an arbitrary physical address space into the kernel virtual - * address space. Needed when the kernel wants to access high addresses - * directly. - * - * NOTE! We need to allow non-page-aligned mappings too: we will obviously - * have to convert them into an offset in a page-aligned mapping, but the - * caller shouldn't need to know that small detail. - */ -void __iomem *__ioremap(unsigned long phys_addr, unsigned long size, - unsigned long flags) -{ - struct vm_struct * area; - unsigned long offset, last_addr, addr, orig_addr; - pgprot_t pgprot; - - /* Don't allow wraparound or zero size */ - last_addr = phys_addr + size - 1; - if (!size || last_addr < phys_addr) - return NULL; - - /* - * If we're on an SH7751 or SH7780 PCI controller, PCI memory is - * mapped at the end of the address space (typically 0xfd000000) - * in a non-translatable area, so mapping through page tables for - * this area is not only pointless, but also fundamentally - * broken. Just return the physical address instead. - * - * For boards that map a small PCI memory aperture somewhere in - * P1/P2 space, ioremap() will already do the right thing, - * and we'll never get this far. - */ - if (is_pci_memaddr(phys_addr) && is_pci_memaddr(last_addr)) - return (void __iomem *)phys_addr; - - /* - * Don't allow anybody to remap normal RAM that we're using.. - */ - if (phys_addr < virt_to_phys(high_memory)) - return NULL; - - /* - * Mappings have to be page-aligned - */ - offset = phys_addr & ~PAGE_MASK; - phys_addr &= PAGE_MASK; - size = PAGE_ALIGN(last_addr+1) - phys_addr; - - /* - * Ok, go for it.. - */ - area = get_vm_area(size, VM_IOREMAP); - if (!area) - return NULL; - area->phys_addr = phys_addr; - orig_addr = addr = (unsigned long)area->addr; - -#ifdef CONFIG_32BIT - /* - * First try to remap through the PMB once a valid VMA has been - * established. Smaller allocations (or the rest of the size - * remaining after a PMB mapping due to the size not being - * perfectly aligned on a PMB size boundary) are then mapped - * through the UTLB using conventional page tables. - * - * PMB entries are all pre-faulted. - */ - if (unlikely(size >= 0x1000000)) { - unsigned long mapped = pmb_remap(addr, phys_addr, size, flags); - - if (likely(mapped)) { - addr += mapped; - phys_addr += mapped; - size -= mapped; - } - } -#endif - - pgprot = __pgprot(pgprot_val(PAGE_KERNEL_NOCACHE) | flags); - if (likely(size)) - if (ioremap_page_range(addr, addr + size, phys_addr, pgprot)) { - vunmap((void *)orig_addr); - return NULL; - } - - return (void __iomem *)(offset + (char *)orig_addr); -} -EXPORT_SYMBOL(__ioremap); - -void __iounmap(void __iomem *addr) -{ - unsigned long vaddr = (unsigned long __force)addr; - struct vm_struct *p; - - if (PXSEG(vaddr) < P3SEG || is_pci_memaddr(vaddr)) - return; - -#ifdef CONFIG_32BIT - /* - * Purge any PMB entries that may have been established for this - * mapping, then proceed with conventional VMA teardown. - * - * XXX: Note that due to the way that remove_vm_area() does - * matching of the resultant VMA, we aren't able to fast-forward - * the address past the PMB space until the end of the VMA where - * the page tables reside. As such, unmap_vm_area() will be - * forced to linearly scan over the area until it finds the page - * tables where PTEs that need to be unmapped actually reside, - * which is far from optimal. Perhaps we need to use a separate - * VMA for the PMB mappings? - * -- PFM. - */ - pmb_unmap(vaddr); -#endif - - p = remove_vm_area((void *)(vaddr & PAGE_MASK)); - if (!p) { - printk(KERN_ERR "%s: bad address %p\n", __FUNCTION__, addr); - return; - } - - kfree(p); -} -EXPORT_SYMBOL(__iounmap); diff --git a/arch/sh/mm/ioremap_32.c b/arch/sh/mm/ioremap_32.c new file mode 100644 index 000000000000..0c7b7e33abdc --- /dev/null +++ b/arch/sh/mm/ioremap_32.c @@ -0,0 +1,150 @@ +/* + * arch/sh/mm/ioremap.c + * + * Re-map IO memory to kernel address space so that we can access it. + * This is needed for high PCI addresses that aren't mapped in the + * 640k-1MB IO memory area on PC's + * + * (C) Copyright 1995 1996 Linus Torvalds + * (C) Copyright 2005, 2006 Paul Mundt + * + * This file is subject to the terms and conditions of the GNU General + * Public License. See the file "COPYING" in the main directory of this + * archive for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Remap an arbitrary physical address space into the kernel virtual + * address space. Needed when the kernel wants to access high addresses + * directly. + * + * NOTE! We need to allow non-page-aligned mappings too: we will obviously + * have to convert them into an offset in a page-aligned mapping, but the + * caller shouldn't need to know that small detail. + */ +void __iomem *__ioremap(unsigned long phys_addr, unsigned long size, + unsigned long flags) +{ + struct vm_struct * area; + unsigned long offset, last_addr, addr, orig_addr; + pgprot_t pgprot; + + /* Don't allow wraparound or zero size */ + last_addr = phys_addr + size - 1; + if (!size || last_addr < phys_addr) + return NULL; + + /* + * If we're on an SH7751 or SH7780 PCI controller, PCI memory is + * mapped at the end of the address space (typically 0xfd000000) + * in a non-translatable area, so mapping through page tables for + * this area is not only pointless, but also fundamentally + * broken. Just return the physical address instead. + * + * For boards that map a small PCI memory aperture somewhere in + * P1/P2 space, ioremap() will already do the right thing, + * and we'll never get this far. + */ + if (is_pci_memaddr(phys_addr) && is_pci_memaddr(last_addr)) + return (void __iomem *)phys_addr; + + /* + * Don't allow anybody to remap normal RAM that we're using.. + */ + if (phys_addr < virt_to_phys(high_memory)) + return NULL; + + /* + * Mappings have to be page-aligned + */ + offset = phys_addr & ~PAGE_MASK; + phys_addr &= PAGE_MASK; + size = PAGE_ALIGN(last_addr+1) - phys_addr; + + /* + * Ok, go for it.. + */ + area = get_vm_area(size, VM_IOREMAP); + if (!area) + return NULL; + area->phys_addr = phys_addr; + orig_addr = addr = (unsigned long)area->addr; + +#ifdef CONFIG_32BIT + /* + * First try to remap through the PMB once a valid VMA has been + * established. Smaller allocations (or the rest of the size + * remaining after a PMB mapping due to the size not being + * perfectly aligned on a PMB size boundary) are then mapped + * through the UTLB using conventional page tables. + * + * PMB entries are all pre-faulted. + */ + if (unlikely(size >= 0x1000000)) { + unsigned long mapped = pmb_remap(addr, phys_addr, size, flags); + + if (likely(mapped)) { + addr += mapped; + phys_addr += mapped; + size -= mapped; + } + } +#endif + + pgprot = __pgprot(pgprot_val(PAGE_KERNEL_NOCACHE) | flags); + if (likely(size)) + if (ioremap_page_range(addr, addr + size, phys_addr, pgprot)) { + vunmap((void *)orig_addr); + return NULL; + } + + return (void __iomem *)(offset + (char *)orig_addr); +} +EXPORT_SYMBOL(__ioremap); + +void __iounmap(void __iomem *addr) +{ + unsigned long vaddr = (unsigned long __force)addr; + struct vm_struct *p; + + if (PXSEG(vaddr) < P3SEG || is_pci_memaddr(vaddr)) + return; + +#ifdef CONFIG_32BIT + /* + * Purge any PMB entries that may have been established for this + * mapping, then proceed with conventional VMA teardown. + * + * XXX: Note that due to the way that remove_vm_area() does + * matching of the resultant VMA, we aren't able to fast-forward + * the address past the PMB space until the end of the VMA where + * the page tables reside. As such, unmap_vm_area() will be + * forced to linearly scan over the area until it finds the page + * tables where PTEs that need to be unmapped actually reside, + * which is far from optimal. Perhaps we need to use a separate + * VMA for the PMB mappings? + * -- PFM. + */ + pmb_unmap(vaddr); +#endif + + p = remove_vm_area((void *)(vaddr & PAGE_MASK)); + if (!p) { + printk(KERN_ERR "%s: bad address %p\n", __FUNCTION__, addr); + return; + } + + kfree(p); +} +EXPORT_SYMBOL(__iounmap); diff --git a/arch/sh/mm/ioremap_64.c b/arch/sh/mm/ioremap_64.c new file mode 100644 index 000000000000..95c85e2e43cf --- /dev/null +++ b/arch/sh/mm/ioremap_64.c @@ -0,0 +1,387 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * arch/sh64/mm/ioremap.c + * + * Copyright (C) 2000, 2001 Paolo Alberelli + * Copyright (C) 2003, 2004 Paul Mundt + * + * Mostly derived from arch/sh/mm/ioremap.c which, in turn is mostly + * derived from arch/i386/mm/ioremap.c . + * + * (C) Copyright 1995 1996 Linus Torvalds + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void shmedia_mapioaddr(unsigned long, unsigned long); +static unsigned long shmedia_ioremap(struct resource *, u32, int); + +/* + * Generic mapping function (not visible outside): + */ + +/* + * Remap an arbitrary physical address space into the kernel virtual + * address space. Needed when the kernel wants to access high addresses + * directly. + * + * NOTE! We need to allow non-page-aligned mappings too: we will obviously + * have to convert them into an offset in a page-aligned mapping, but the + * caller shouldn't need to know that small detail. + */ +void * __ioremap(unsigned long phys_addr, unsigned long size, unsigned long flags) +{ + void * addr; + struct vm_struct * area; + unsigned long offset, last_addr; + pgprot_t pgprot; + + /* Don't allow wraparound or zero size */ + last_addr = phys_addr + size - 1; + if (!size || last_addr < phys_addr) + return NULL; + + pgprot = __pgprot(_PAGE_PRESENT | _PAGE_READ | + _PAGE_WRITE | _PAGE_DIRTY | + _PAGE_ACCESSED | _PAGE_SHARED | flags); + + /* + * Mappings have to be page-aligned + */ + offset = phys_addr & ~PAGE_MASK; + phys_addr &= PAGE_MASK; + size = PAGE_ALIGN(last_addr + 1) - phys_addr; + + /* + * Ok, go for it.. + */ + area = get_vm_area(size, VM_IOREMAP); + pr_debug("Get vm_area returns %p addr %p\n",area,area->addr); + if (!area) + return NULL; + area->phys_addr = phys_addr; + addr = area->addr; + if (ioremap_page_range((unsigned long)addr, (unsigned long)addr + size, + phys_addr, pgprot)) { + vunmap(addr); + return NULL; + } + return (void *) (offset + (char *)addr); +} +EXPORT_SYMBOL(__ioremap); + +void iounmap(void *addr) +{ + struct vm_struct *area; + + vfree((void *) (PAGE_MASK & (unsigned long) addr)); + area = remove_vm_area((void *) (PAGE_MASK & (unsigned long) addr)); + if (!area) { + printk(KERN_ERR "iounmap: bad address %p\n", addr); + return; + } + + kfree(area); +} +EXPORT_SYMBOL(iounmap); + +static struct resource shmedia_iomap = { + .name = "shmedia_iomap", + .start = IOBASE_VADDR + PAGE_SIZE, + .end = IOBASE_END - 1, +}; + +static void shmedia_mapioaddr(unsigned long pa, unsigned long va); +static void shmedia_unmapioaddr(unsigned long vaddr); +static unsigned long shmedia_ioremap(struct resource *res, u32 pa, int sz); + +/* + * We have the same problem as the SPARC, so lets have the same comment: + * Our mini-allocator... + * Boy this is gross! We need it because we must map I/O for + * timers and interrupt controller before the kmalloc is available. + */ + +#define XNMLN 15 +#define XNRES 10 + +struct xresource { + struct resource xres; /* Must be first */ + int xflag; /* 1 == used */ + char xname[XNMLN+1]; +}; + +static struct xresource xresv[XNRES]; + +static struct xresource *xres_alloc(void) +{ + struct xresource *xrp; + int n; + + xrp = xresv; + for (n = 0; n < XNRES; n++) { + if (xrp->xflag == 0) { + xrp->xflag = 1; + return xrp; + } + xrp++; + } + return NULL; +} + +static void xres_free(struct xresource *xrp) +{ + xrp->xflag = 0; +} + +static struct resource *shmedia_find_resource(struct resource *root, + unsigned long vaddr) +{ + struct resource *res; + + for (res = root->child; res; res = res->sibling) + if (res->start <= vaddr && res->end >= vaddr) + return res; + + return NULL; +} + +static unsigned long shmedia_alloc_io(unsigned long phys, unsigned long size, + const char *name) +{ + static int printed_full = 0; + struct xresource *xres; + struct resource *res; + char *tack; + int tlen; + + if (name == NULL) name = "???"; + + if ((xres = xres_alloc()) != 0) { + tack = xres->xname; + res = &xres->xres; + } else { + if (!printed_full) { + printk("%s: done with statics, switching to kmalloc\n", + __FUNCTION__); + printed_full = 1; + } + tlen = strlen(name); + tack = kmalloc(sizeof (struct resource) + tlen + 1, GFP_KERNEL); + if (!tack) + return -ENOMEM; + memset(tack, 0, sizeof(struct resource)); + res = (struct resource *) tack; + tack += sizeof (struct resource); + } + + strncpy(tack, name, XNMLN); + tack[XNMLN] = 0; + res->name = tack; + + return shmedia_ioremap(res, phys, size); +} + +static unsigned long shmedia_ioremap(struct resource *res, u32 pa, int sz) +{ + unsigned long offset = ((unsigned long) pa) & (~PAGE_MASK); + unsigned long round_sz = (offset + sz + PAGE_SIZE-1) & PAGE_MASK; + unsigned long va; + unsigned int psz; + + if (allocate_resource(&shmedia_iomap, res, round_sz, + shmedia_iomap.start, shmedia_iomap.end, + PAGE_SIZE, NULL, NULL) != 0) { + panic("alloc_io_res(%s): cannot occupy\n", + (res->name != NULL)? res->name: "???"); + } + + va = res->start; + pa &= PAGE_MASK; + + psz = (res->end - res->start + (PAGE_SIZE - 1)) / PAGE_SIZE; + + /* log at boot time ... */ + printk("mapioaddr: %6s [%2d page%s] va 0x%08lx pa 0x%08x\n", + ((res->name != NULL) ? res->name : "???"), + psz, psz == 1 ? " " : "s", va, pa); + + for (psz = res->end - res->start + 1; psz != 0; psz -= PAGE_SIZE) { + shmedia_mapioaddr(pa, va); + va += PAGE_SIZE; + pa += PAGE_SIZE; + } + + res->start += offset; + res->end = res->start + sz - 1; /* not strictly necessary.. */ + + return res->start; +} + +static void shmedia_free_io(struct resource *res) +{ + unsigned long len = res->end - res->start + 1; + + BUG_ON((len & (PAGE_SIZE - 1)) != 0); + + while (len) { + len -= PAGE_SIZE; + shmedia_unmapioaddr(res->start + len); + } + + release_resource(res); +} + +static __init_refok void *sh64_get_page(void) +{ + extern int after_bootmem; + void *page; + + if (after_bootmem) { + page = (void *)get_zeroed_page(GFP_ATOMIC); + } else { + page = alloc_bootmem_pages(PAGE_SIZE); + } + + if (!page || ((unsigned long)page & ~PAGE_MASK)) + panic("sh64_get_page: Out of memory already?\n"); + + return page; +} + +static void shmedia_mapioaddr(unsigned long pa, unsigned long va) +{ + pgd_t *pgdp; + pmd_t *pmdp; + pte_t *ptep, pte; + pgprot_t prot; + unsigned long flags = 1; /* 1 = CB0-1 device */ + + pr_debug("shmedia_mapiopage pa %08lx va %08lx\n", pa, va); + + pgdp = pgd_offset_k(va); + if (pgd_none(*pgdp) || !pgd_present(*pgdp)) { + pmdp = (pmd_t *)sh64_get_page(); + set_pgd(pgdp, __pgd((unsigned long)pmdp | _KERNPG_TABLE)); + } + + pmdp = pmd_offset(pgdp, va); + if (pmd_none(*pmdp) || !pmd_present(*pmdp) ) { + ptep = (pte_t *)sh64_get_page(); + set_pmd(pmdp, __pmd((unsigned long)ptep + _PAGE_TABLE)); + } + + prot = __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | + _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_SHARED | flags); + + pte = pfn_pte(pa >> PAGE_SHIFT, prot); + ptep = pte_offset_kernel(pmdp, va); + + if (!pte_none(*ptep) && + pte_val(*ptep) != pte_val(pte)) + pte_ERROR(*ptep); + + set_pte(ptep, pte); + + flush_tlb_kernel_range(va, PAGE_SIZE); +} + +static void shmedia_unmapioaddr(unsigned long vaddr) +{ + pgd_t *pgdp; + pmd_t *pmdp; + pte_t *ptep; + + pgdp = pgd_offset_k(vaddr); + pmdp = pmd_offset(pgdp, vaddr); + + if (pmd_none(*pmdp) || pmd_bad(*pmdp)) + return; + + ptep = pte_offset_kernel(pmdp, vaddr); + + if (pte_none(*ptep) || !pte_present(*ptep)) + return; + + clear_page((void *)ptep); + pte_clear(&init_mm, vaddr, ptep); +} + +unsigned long onchip_remap(unsigned long phys, unsigned long size, const char *name) +{ + if (size < PAGE_SIZE) + size = PAGE_SIZE; + + return shmedia_alloc_io(phys, size, name); +} + +void onchip_unmap(unsigned long vaddr) +{ + struct resource *res; + unsigned int psz; + + res = shmedia_find_resource(&shmedia_iomap, vaddr); + if (!res) { + printk(KERN_ERR "%s: Failed to free 0x%08lx\n", + __FUNCTION__, vaddr); + return; + } + + psz = (res->end - res->start + (PAGE_SIZE - 1)) / PAGE_SIZE; + + printk(KERN_DEBUG "unmapioaddr: %6s [%2d page%s] freed\n", + res->name, psz, psz == 1 ? " " : "s"); + + shmedia_free_io(res); + + if ((char *)res >= (char *)xresv && + (char *)res < (char *)&xresv[XNRES]) { + xres_free((struct xresource *)res); + } else { + kfree(res); + } +} + +#ifdef CONFIG_PROC_FS +static int +ioremap_proc_info(char *buf, char **start, off_t fpos, int length, int *eof, + void *data) +{ + char *p = buf, *e = buf + length; + struct resource *r; + const char *nm; + + for (r = ((struct resource *)data)->child; r != NULL; r = r->sibling) { + if (p + 32 >= e) /* Better than nothing */ + break; + if ((nm = r->name) == 0) nm = "???"; + p += sprintf(p, "%08lx-%08lx: %s\n", + (unsigned long)r->start, + (unsigned long)r->end, nm); + } + + return p-buf; +} +#endif /* CONFIG_PROC_FS */ + +static int __init register_proc_onchip(void) +{ +#ifdef CONFIG_PROC_FS + create_proc_read_entry("io_map",0,0, ioremap_proc_info, &shmedia_iomap); +#endif + return 0; +} + +__initcall(register_proc_onchip); -- cgit v1.2.3 From 2b6a8d455b1368d769da234336314b8364feb781 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Mon, 19 Nov 2007 14:26:09 +0900 Subject: sh: ioremap_64 needs proc_fs.h. Signed-off-by: Paul Mundt --- arch/sh/mm/ioremap_64.c | 1 + 1 file changed, 1 insertion(+) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/ioremap_64.c b/arch/sh/mm/ioremap_64.c index 95c85e2e43cf..95462a0f3120 100644 --- a/arch/sh/mm/ioremap_64.c +++ b/arch/sh/mm/ioremap_64.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include -- cgit v1.2.3 From 249cfea914002baac0af4b080306e6b820cd86b2 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Mon, 19 Nov 2007 18:26:19 +0900 Subject: sh: Split out pgtable.h in to _32 and _64 variants. Signed-off-by: Paul Mundt --- arch/sh/mm/ioremap_64.c | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/ioremap_64.c b/arch/sh/mm/ioremap_64.c index 95462a0f3120..e27d16519235 100644 --- a/arch/sh/mm/ioremap_64.c +++ b/arch/sh/mm/ioremap_64.c @@ -1,23 +1,24 @@ /* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * arch/sh64/mm/ioremap.c + * arch/sh/mm/ioremap_64.c * * Copyright (C) 2000, 2001 Paolo Alberelli - * Copyright (C) 2003, 2004 Paul Mundt + * Copyright (C) 2003 - 2007 Paul Mundt * * Mostly derived from arch/sh/mm/ioremap.c which, in turn is mostly * derived from arch/i386/mm/ioremap.c . * * (C) Copyright 1995 1996 Linus Torvalds + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. */ #include #include #include #include #include +#include #include #include #include @@ -42,7 +43,8 @@ static unsigned long shmedia_ioremap(struct resource *, u32, int); * have to convert them into an offset in a page-aligned mapping, but the * caller shouldn't need to know that small detail. */ -void * __ioremap(unsigned long phys_addr, unsigned long size, unsigned long flags) +void *__ioremap(unsigned long phys_addr, unsigned long size, + unsigned long flags) { void * addr; struct vm_struct * area; @@ -83,7 +85,7 @@ void * __ioremap(unsigned long phys_addr, unsigned long size, unsigned long flag } EXPORT_SYMBOL(__ioremap); -void iounmap(void *addr) +void __iounmap(void *addr) { struct vm_struct *area; @@ -96,7 +98,7 @@ void iounmap(void *addr) kfree(area); } -EXPORT_SYMBOL(iounmap); +EXPORT_SYMBOL(__iounmap); static struct resource shmedia_iomap = { .name = "shmedia_iomap", @@ -265,6 +267,7 @@ static __init_refok void *sh64_get_page(void) static void shmedia_mapioaddr(unsigned long pa, unsigned long va) { pgd_t *pgdp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep, pte; pgprot_t prot; @@ -274,11 +277,17 @@ static void shmedia_mapioaddr(unsigned long pa, unsigned long va) pgdp = pgd_offset_k(va); if (pgd_none(*pgdp) || !pgd_present(*pgdp)) { + pudp = (pud_t *)sh64_get_page(); + set_pgd(pgdp, __pgd((unsigned long)pudp | _KERNPG_TABLE)); + } + + pudp = pud_offset(pgdp, va); + if (pud_none(*pudp) || !pud_present(*pudp)) { pmdp = (pmd_t *)sh64_get_page(); - set_pgd(pgdp, __pgd((unsigned long)pmdp | _KERNPG_TABLE)); + set_pud(pudp, __pud((unsigned long)pmdp | _KERNPG_TABLE)); } - pmdp = pmd_offset(pgdp, va); + pmdp = pmd_offset(pudp, va); if (pmd_none(*pmdp) || !pmd_present(*pmdp) ) { ptep = (pte_t *)sh64_get_page(); set_pmd(pmdp, __pmd((unsigned long)ptep + _PAGE_TABLE)); @@ -302,12 +311,19 @@ static void shmedia_mapioaddr(unsigned long pa, unsigned long va) static void shmedia_unmapioaddr(unsigned long vaddr) { pgd_t *pgdp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; pgdp = pgd_offset_k(vaddr); - pmdp = pmd_offset(pgdp, vaddr); + if (pgd_none(*pgdp) || pgd_bad(*pgdp)) + return; + + pudp = pud_offset(pgdp, vaddr); + if (pud_none(*pudp) || pud_bad(*pudp)) + return; + pmdp = pmd_offset(pudp, vaddr); if (pmd_none(*pmdp) || pmd_bad(*pmdp)) return; -- cgit v1.2.3 From 3eeffb32049d4b43b01211a79496cfdaac0c33ad Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Mon, 19 Nov 2007 18:57:03 +0900 Subject: sh: Split out tlb-flush in to _32 and _64 variants. Signed-off-by: Paul Mundt --- arch/sh/mm/Makefile_32 | 2 +- arch/sh/mm/Makefile_64 | 2 +- arch/sh/mm/tlb-flush.c | 140 -------------- arch/sh/mm/tlb-flush_32.c | 140 ++++++++++++++ arch/sh/mm/tlb-flush_64.c | 475 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 617 insertions(+), 142 deletions(-) delete mode 100644 arch/sh/mm/tlb-flush.c create mode 100644 arch/sh/mm/tlb-flush_32.c create mode 100644 arch/sh/mm/tlb-flush_64.c (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/Makefile_32 b/arch/sh/mm/Makefile_32 index b43884269c2a..9ea379303742 100644 --- a/arch/sh/mm/Makefile_32 +++ b/arch/sh/mm/Makefile_32 @@ -12,7 +12,7 @@ obj-$(CONFIG_SH7705_CACHE_32KB) += cache-sh7705.o endif mmu-y := tlb-nommu.o pg-nommu.o -mmu-$(CONFIG_MMU) := fault.o clear_page.o copy_page.o tlb-flush.o \ +mmu-$(CONFIG_MMU) := fault.o clear_page.o copy_page.o tlb-flush_32.o \ ioremap_32.o obj-y += $(mmu-y) diff --git a/arch/sh/mm/Makefile_64 b/arch/sh/mm/Makefile_64 index 177f92863e4e..ec8deaa17909 100644 --- a/arch/sh/mm/Makefile_64 +++ b/arch/sh/mm/Makefile_64 @@ -5,7 +5,7 @@ obj-y := init.o extable_64.o consistent.o mmu-y := tlb-nommu.o pg-nommu.o -mmu-$(CONFIG_MMU) := ioremap_64.o +mmu-$(CONFIG_MMU) := ioremap_64.o tlb-flush_64.o obj-y += $(mmu-y) diff --git a/arch/sh/mm/tlb-flush.c b/arch/sh/mm/tlb-flush.c deleted file mode 100644 index 6f45c1f8a7fe..000000000000 --- a/arch/sh/mm/tlb-flush.c +++ /dev/null @@ -1,140 +0,0 @@ -/* - * TLB flushing operations for SH with an MMU. - * - * Copyright (C) 1999 Niibe Yutaka - * Copyright (C) 2003 Paul Mundt - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - */ -#include -#include -#include - -void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) -{ - unsigned int cpu = smp_processor_id(); - - if (vma->vm_mm && cpu_context(cpu, vma->vm_mm) != NO_CONTEXT) { - unsigned long flags; - unsigned long asid; - unsigned long saved_asid = MMU_NO_ASID; - - asid = cpu_asid(cpu, vma->vm_mm); - page &= PAGE_MASK; - - local_irq_save(flags); - if (vma->vm_mm != current->mm) { - saved_asid = get_asid(); - set_asid(asid); - } - local_flush_tlb_one(asid, page); - if (saved_asid != MMU_NO_ASID) - set_asid(saved_asid); - local_irq_restore(flags); - } -} - -void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end) -{ - struct mm_struct *mm = vma->vm_mm; - unsigned int cpu = smp_processor_id(); - - if (cpu_context(cpu, mm) != NO_CONTEXT) { - unsigned long flags; - int size; - - local_irq_save(flags); - size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; - if (size > (MMU_NTLB_ENTRIES/4)) { /* Too many TLB to flush */ - cpu_context(cpu, mm) = NO_CONTEXT; - if (mm == current->mm) - activate_context(mm, cpu); - } else { - unsigned long asid; - unsigned long saved_asid = MMU_NO_ASID; - - asid = cpu_asid(cpu, mm); - start &= PAGE_MASK; - end += (PAGE_SIZE - 1); - end &= PAGE_MASK; - if (mm != current->mm) { - saved_asid = get_asid(); - set_asid(asid); - } - while (start < end) { - local_flush_tlb_one(asid, start); - start += PAGE_SIZE; - } - if (saved_asid != MMU_NO_ASID) - set_asid(saved_asid); - } - local_irq_restore(flags); - } -} - -void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) -{ - unsigned int cpu = smp_processor_id(); - unsigned long flags; - int size; - - local_irq_save(flags); - size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; - if (size > (MMU_NTLB_ENTRIES/4)) { /* Too many TLB to flush */ - local_flush_tlb_all(); - } else { - unsigned long asid; - unsigned long saved_asid = get_asid(); - - asid = cpu_asid(cpu, &init_mm); - start &= PAGE_MASK; - end += (PAGE_SIZE - 1); - end &= PAGE_MASK; - set_asid(asid); - while (start < end) { - local_flush_tlb_one(asid, start); - start += PAGE_SIZE; - } - set_asid(saved_asid); - } - local_irq_restore(flags); -} - -void local_flush_tlb_mm(struct mm_struct *mm) -{ - unsigned int cpu = smp_processor_id(); - - /* Invalidate all TLB of this process. */ - /* Instead of invalidating each TLB, we get new MMU context. */ - if (cpu_context(cpu, mm) != NO_CONTEXT) { - unsigned long flags; - - local_irq_save(flags); - cpu_context(cpu, mm) = NO_CONTEXT; - if (mm == current->mm) - activate_context(mm, cpu); - local_irq_restore(flags); - } -} - -void local_flush_tlb_all(void) -{ - unsigned long flags, status; - - /* - * Flush all the TLB. - * - * Write to the MMU control register's bit: - * TF-bit for SH-3, TI-bit for SH-4. - * It's same position, bit #2. - */ - local_irq_save(flags); - status = ctrl_inl(MMUCR); - status |= 0x04; - ctrl_outl(status, MMUCR); - ctrl_barrier(); - local_irq_restore(flags); -} diff --git a/arch/sh/mm/tlb-flush_32.c b/arch/sh/mm/tlb-flush_32.c new file mode 100644 index 000000000000..6f45c1f8a7fe --- /dev/null +++ b/arch/sh/mm/tlb-flush_32.c @@ -0,0 +1,140 @@ +/* + * TLB flushing operations for SH with an MMU. + * + * Copyright (C) 1999 Niibe Yutaka + * Copyright (C) 2003 Paul Mundt + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include +#include +#include + +void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + unsigned int cpu = smp_processor_id(); + + if (vma->vm_mm && cpu_context(cpu, vma->vm_mm) != NO_CONTEXT) { + unsigned long flags; + unsigned long asid; + unsigned long saved_asid = MMU_NO_ASID; + + asid = cpu_asid(cpu, vma->vm_mm); + page &= PAGE_MASK; + + local_irq_save(flags); + if (vma->vm_mm != current->mm) { + saved_asid = get_asid(); + set_asid(asid); + } + local_flush_tlb_one(asid, page); + if (saved_asid != MMU_NO_ASID) + set_asid(saved_asid); + local_irq_restore(flags); + } +} + +void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned int cpu = smp_processor_id(); + + if (cpu_context(cpu, mm) != NO_CONTEXT) { + unsigned long flags; + int size; + + local_irq_save(flags); + size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; + if (size > (MMU_NTLB_ENTRIES/4)) { /* Too many TLB to flush */ + cpu_context(cpu, mm) = NO_CONTEXT; + if (mm == current->mm) + activate_context(mm, cpu); + } else { + unsigned long asid; + unsigned long saved_asid = MMU_NO_ASID; + + asid = cpu_asid(cpu, mm); + start &= PAGE_MASK; + end += (PAGE_SIZE - 1); + end &= PAGE_MASK; + if (mm != current->mm) { + saved_asid = get_asid(); + set_asid(asid); + } + while (start < end) { + local_flush_tlb_one(asid, start); + start += PAGE_SIZE; + } + if (saved_asid != MMU_NO_ASID) + set_asid(saved_asid); + } + local_irq_restore(flags); + } +} + +void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + unsigned int cpu = smp_processor_id(); + unsigned long flags; + int size; + + local_irq_save(flags); + size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; + if (size > (MMU_NTLB_ENTRIES/4)) { /* Too many TLB to flush */ + local_flush_tlb_all(); + } else { + unsigned long asid; + unsigned long saved_asid = get_asid(); + + asid = cpu_asid(cpu, &init_mm); + start &= PAGE_MASK; + end += (PAGE_SIZE - 1); + end &= PAGE_MASK; + set_asid(asid); + while (start < end) { + local_flush_tlb_one(asid, start); + start += PAGE_SIZE; + } + set_asid(saved_asid); + } + local_irq_restore(flags); +} + +void local_flush_tlb_mm(struct mm_struct *mm) +{ + unsigned int cpu = smp_processor_id(); + + /* Invalidate all TLB of this process. */ + /* Instead of invalidating each TLB, we get new MMU context. */ + if (cpu_context(cpu, mm) != NO_CONTEXT) { + unsigned long flags; + + local_irq_save(flags); + cpu_context(cpu, mm) = NO_CONTEXT; + if (mm == current->mm) + activate_context(mm, cpu); + local_irq_restore(flags); + } +} + +void local_flush_tlb_all(void) +{ + unsigned long flags, status; + + /* + * Flush all the TLB. + * + * Write to the MMU control register's bit: + * TF-bit for SH-3, TI-bit for SH-4. + * It's same position, bit #2. + */ + local_irq_save(flags); + status = ctrl_inl(MMUCR); + status |= 0x04; + ctrl_outl(status, MMUCR); + ctrl_barrier(); + local_irq_restore(flags); +} diff --git a/arch/sh/mm/tlb-flush_64.c b/arch/sh/mm/tlb-flush_64.c new file mode 100644 index 000000000000..2a98c9ec88ff --- /dev/null +++ b/arch/sh/mm/tlb-flush_64.c @@ -0,0 +1,475 @@ +/* + * arch/sh/mm/tlb-flush_64.c + * + * Copyright (C) 2000, 2001 Paolo Alberelli + * Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes) + * Copyright (C) 2003 Paul Mundt + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern void die(const char *,struct pt_regs *,long); + +#define PFLAG(val,flag) (( (val) & (flag) ) ? #flag : "" ) +#define PPROT(flag) PFLAG(pgprot_val(prot),flag) + +static inline void print_prots(pgprot_t prot) +{ + printk("prot is 0x%08lx\n",pgprot_val(prot)); + + printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ), + PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER)); +} + +static inline void print_vma(struct vm_area_struct *vma) +{ + printk("vma start 0x%08lx\n", vma->vm_start); + printk("vma end 0x%08lx\n", vma->vm_end); + + print_prots(vma->vm_page_prot); + printk("vm_flags 0x%08lx\n", vma->vm_flags); +} + +static inline void print_task(struct task_struct *tsk) +{ + printk("Task pid %d\n", task_pid_nr(tsk)); +} + +static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address) +{ + pgd_t *dir; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + pte_t entry; + + dir = pgd_offset(mm, address); + if (pgd_none(*dir)) + return NULL; + + pud = pud_offset(dir, address); + if (pud_none(*pud)) + return NULL; + + pmd = pmd_offset(pud, address); + if (pmd_none(*pmd)) + return NULL; + + pte = pte_offset_kernel(pmd, address); + entry = *pte; + if (pte_none(entry) || !pte_present(entry)) + return NULL; + + return pte; +} + +/* + * This routine handles page faults. It determines the address, + * and the problem, and then passes it off to one of the appropriate + * routines. + */ +asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess, + unsigned long textaccess, unsigned long address) +{ + struct task_struct *tsk; + struct mm_struct *mm; + struct vm_area_struct * vma; + const struct exception_table_entry *fixup; + pte_t *pte; + int fault; + + /* SIM + * Note this is now called with interrupts still disabled + * This is to cope with being called for a missing IO port + * address with interrupts disabled. This should be fixed as + * soon as we have a better 'fast path' miss handler. + * + * Plus take care how you try and debug this stuff. + * For example, writing debug data to a port which you + * have just faulted on is not going to work. + */ + + tsk = current; + mm = tsk->mm; + + /* Not an IO address, so reenable interrupts */ + local_irq_enable(); + + /* + * If we're in an interrupt or have no user + * context, we must not take the fault.. + */ + if (in_atomic() || !mm) + goto no_context; + + /* TLB misses upon some cache flushes get done under cli() */ + down_read(&mm->mmap_sem); + + vma = find_vma(mm, address); + + if (!vma) { +#ifdef DEBUG_FAULT + print_task(tsk); + printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", + __FUNCTION__,__LINE__, + address,regs->pc,textaccess,writeaccess); + show_regs(regs); +#endif + goto bad_area; + } + if (vma->vm_start <= address) { + goto good_area; + } + + if (!(vma->vm_flags & VM_GROWSDOWN)) { +#ifdef DEBUG_FAULT + print_task(tsk); + printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", + __FUNCTION__,__LINE__, + address,regs->pc,textaccess,writeaccess); + show_regs(regs); + + print_vma(vma); +#endif + goto bad_area; + } + if (expand_stack(vma, address)) { +#ifdef DEBUG_FAULT + print_task(tsk); + printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", + __FUNCTION__,__LINE__, + address,regs->pc,textaccess,writeaccess); + show_regs(regs); +#endif + goto bad_area; + } +/* + * Ok, we have a good vm_area for this memory access, so + * we can handle it.. + */ +good_area: + if (textaccess) { + if (!(vma->vm_flags & VM_EXEC)) + goto bad_area; + } else { + if (writeaccess) { + if (!(vma->vm_flags & VM_WRITE)) + goto bad_area; + } else { + if (!(vma->vm_flags & VM_READ)) + goto bad_area; + } + } + + /* + * If for any reason at all we couldn't handle the fault, + * make sure we exit gracefully rather than endlessly redo + * the fault. + */ +survive: + fault = handle_mm_fault(mm, vma, address, writeaccess); + if (unlikely(fault & VM_FAULT_ERROR)) { + if (fault & VM_FAULT_OOM) + goto out_of_memory; + else if (fault & VM_FAULT_SIGBUS) + goto do_sigbus; + BUG(); + } + if (fault & VM_FAULT_MAJOR) + tsk->maj_flt++; + else + tsk->min_flt++; + + /* If we get here, the page fault has been handled. Do the TLB refill + now from the newly-setup PTE, to avoid having to fault again right + away on the same instruction. */ + pte = lookup_pte (mm, address); + if (!pte) { + /* From empirical evidence, we can get here, due to + !pte_present(pte). (e.g. if a swap-in occurs, and the page + is swapped back out again before the process that wanted it + gets rescheduled?) */ + goto no_pte; + } + + __do_tlb_refill(address, textaccess, pte); + +no_pte: + + up_read(&mm->mmap_sem); + return; + +/* + * Something tried to access memory that isn't in our memory map.. + * Fix it, but check if it's kernel or user first.. + */ +bad_area: +#ifdef DEBUG_FAULT + printk("fault:bad area\n"); +#endif + up_read(&mm->mmap_sem); + + if (user_mode(regs)) { + static int count=0; + siginfo_t info; + if (count < 4) { + /* This is really to help debug faults when starting + * usermode, so only need a few */ + count++; + printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n", + address, task_pid_nr(current), current->comm, + (unsigned long) regs->pc); +#if 0 + show_regs(regs); +#endif + } + if (is_global_init(tsk)) { + panic("INIT had user mode bad_area\n"); + } + tsk->thread.address = address; + tsk->thread.error_code = writeaccess; + info.si_signo = SIGSEGV; + info.si_errno = 0; + info.si_addr = (void *) address; + force_sig_info(SIGSEGV, &info, tsk); + return; + } + +no_context: +#ifdef DEBUG_FAULT + printk("fault:No context\n"); +#endif + /* Are we prepared to handle this kernel fault? */ + fixup = search_exception_tables(regs->pc); + if (fixup) { + regs->pc = fixup->fixup; + return; + } + +/* + * Oops. The kernel tried to access some bad page. We'll have to + * terminate things with extreme prejudice. + * + */ + if (address < PAGE_SIZE) + printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference"); + else + printk(KERN_ALERT "Unable to handle kernel paging request"); + printk(" at virtual address %08lx\n", address); + printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff); + die("Oops", regs, writeaccess); + do_exit(SIGKILL); + +/* + * We ran out of memory, or some other thing happened to us that made + * us unable to handle the page fault gracefully. + */ +out_of_memory: + if (is_global_init(current)) { + panic("INIT out of memory\n"); + yield(); + goto survive; + } + printk("fault:Out of memory\n"); + up_read(&mm->mmap_sem); + if (is_global_init(current)) { + yield(); + down_read(&mm->mmap_sem); + goto survive; + } + printk("VM: killing process %s\n", tsk->comm); + if (user_mode(regs)) + do_group_exit(SIGKILL); + goto no_context; + +do_sigbus: + printk("fault:Do sigbus\n"); + up_read(&mm->mmap_sem); + + /* + * Send a sigbus, regardless of whether we were in kernel + * or user mode. + */ + tsk->thread.address = address; + tsk->thread.error_code = writeaccess; + tsk->thread.trap_no = 14; + force_sig(SIGBUS, tsk); + + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) + goto no_context; +} + +void update_mmu_cache(struct vm_area_struct * vma, + unsigned long address, pte_t pte) +{ + /* + * This appears to get called once for every pte entry that gets + * established => I don't think it's efficient to try refilling the + * TLBs with the pages - some may not get accessed even. Also, for + * executable pages, it is impossible to determine reliably here which + * TLB they should be mapped into (or both even). + * + * So, just do nothing here and handle faults on demand. In the + * TLBMISS handling case, the refill is now done anyway after the pte + * has been fixed up, so that deals with most useful cases. + */ +} + +void local_flush_tlb_one(unsigned long asid, unsigned long page) +{ + unsigned long long match, pteh=0, lpage; + unsigned long tlb; + + /* + * Sign-extend based on neff. + */ + lpage = (page & NEFF_SIGN) ? (page | NEFF_MASK) : page; + match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID; + match |= lpage; + + for_each_itlb_entry(tlb) { + asm volatile ("getcfg %1, 0, %0" + : "=r" (pteh) + : "r" (tlb) ); + + if (pteh == match) { + __flush_tlb_slot(tlb); + break; + } + } + + for_each_dtlb_entry(tlb) { + asm volatile ("getcfg %1, 0, %0" + : "=r" (pteh) + : "r" (tlb) ); + + if (pteh == match) { + __flush_tlb_slot(tlb); + break; + } + + } +} + +void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + unsigned long flags; + + if (vma->vm_mm) { + page &= PAGE_MASK; + local_irq_save(flags); + local_flush_tlb_one(get_asid(), page); + local_irq_restore(flags); + } +} + +void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) +{ + unsigned long flags; + unsigned long long match, pteh=0, pteh_epn, pteh_low; + unsigned long tlb; + unsigned int cpu = smp_processor_id(); + struct mm_struct *mm; + + mm = vma->vm_mm; + if (cpu_context(cpu, mm) == NO_CONTEXT) + return; + + local_irq_save(flags); + + start &= PAGE_MASK; + end &= PAGE_MASK; + + match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID; + + /* Flush ITLB */ + for_each_itlb_entry(tlb) { + asm volatile ("getcfg %1, 0, %0" + : "=r" (pteh) + : "r" (tlb) ); + + pteh_epn = pteh & PAGE_MASK; + pteh_low = pteh & ~PAGE_MASK; + + if (pteh_low == match && pteh_epn >= start && pteh_epn <= end) + __flush_tlb_slot(tlb); + } + + /* Flush DTLB */ + for_each_dtlb_entry(tlb) { + asm volatile ("getcfg %1, 0, %0" + : "=r" (pteh) + : "r" (tlb) ); + + pteh_epn = pteh & PAGE_MASK; + pteh_low = pteh & ~PAGE_MASK; + + if (pteh_low == match && pteh_epn >= start && pteh_epn <= end) + __flush_tlb_slot(tlb); + } + + local_irq_restore(flags); +} + +void local_flush_tlb_mm(struct mm_struct *mm) +{ + unsigned long flags; + unsigned int cpu = smp_processor_id(); + + if (cpu_context(cpu, mm) == NO_CONTEXT) + return; + + local_irq_save(flags); + + cpu_context(cpu, mm) = NO_CONTEXT; + if (mm == current->mm) + activate_context(mm, cpu); + + local_irq_restore(flags); +} + +void local_flush_tlb_all(void) +{ + /* Invalidate all, including shared pages, excluding fixed TLBs */ + unsigned long flags, tlb; + + local_irq_save(flags); + + /* Flush each ITLB entry */ + for_each_itlb_entry(tlb) + __flush_tlb_slot(tlb); + + /* Flush each DTLB entry */ + for_each_dtlb_entry(tlb) + __flush_tlb_slot(tlb); + + local_irq_restore(flags); +} + +void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + /* FIXME: Optimize this later.. */ + flush_tlb_all(); +} -- cgit v1.2.3 From ba2727b556778f8af7cf08956d84723930a83965 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Tue, 20 Nov 2007 15:14:48 +0900 Subject: sh: ioremap_64 needs after_bootmem. Signed-off-by: Paul Mundt --- arch/sh/mm/init.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index d5e160da64b2..5d19c8c7ab53 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -170,6 +170,7 @@ void __init paging_init(void) } static struct kcore_list kcore_mem, kcore_vmalloc; +int after_bootmem = 0; void __init mem_init(void) { @@ -214,6 +215,8 @@ void __init mem_init(void) clear_page = clear_page_nommu; #endif + after_bootmem = 1; + codesize = (unsigned long) &_etext - (unsigned long) &_text; datasize = (unsigned long) &_edata - (unsigned long) &_etext; initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin; -- cgit v1.2.3 From 379a95d1d2c3e3682e380084c40b6fc01e38fa1f Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Tue, 20 Nov 2007 16:51:28 +0900 Subject: sh: Tidy up various clear_page()/copy_page() definitions. Signed-off-by: Paul Mundt --- arch/sh/mm/copy_page.S | 6 +++--- arch/sh/mm/init.c | 15 --------------- arch/sh/mm/pg-nommu.c | 4 ++-- 3 files changed, 5 insertions(+), 20 deletions(-) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/copy_page.S b/arch/sh/mm/copy_page.S index 40685018b952..b879545fa28b 100644 --- a/arch/sh/mm/copy_page.S +++ b/arch/sh/mm/copy_page.S @@ -9,11 +9,11 @@ #include /* - * copy_page_slow + * copy_page * @to: P1 address * @from: P1 address * - * void copy_page_slow(void *to, void *from) + * void copy_page(void *to, void *from) */ /* @@ -23,7 +23,7 @@ * r10 --- to * r11 --- from */ -ENTRY(copy_page_slow) +ENTRY(copy_page) mov.l r8,@-r15 mov.l r10,@-r15 mov.l r11,@-r15 diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index 5d19c8c7ab53..79c309780f95 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -24,9 +24,6 @@ DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); pgd_t swapper_pg_dir[PTRS_PER_PGD]; -void (*copy_page)(void *from, void *to); -void (*clear_page)(void *to); - void show_mem(void) { int total = 0, reserved = 0, free = 0; @@ -203,18 +200,6 @@ void __init mem_init(void) memset(empty_zero_page, 0, PAGE_SIZE); __flush_wback_region(empty_zero_page, PAGE_SIZE); - /* - * Setup wrappers for copy/clear_page(), these will get overridden - * later in the boot process if a better method is available. - */ -#ifdef CONFIG_MMU - copy_page = copy_page_slow; - clear_page = clear_page_slow; -#else - copy_page = copy_page_nommu; - clear_page = clear_page_nommu; -#endif - after_bootmem = 1; codesize = (unsigned long) &_etext - (unsigned long) &_text; diff --git a/arch/sh/mm/pg-nommu.c b/arch/sh/mm/pg-nommu.c index d15221beaa16..677dd57f0877 100644 --- a/arch/sh/mm/pg-nommu.c +++ b/arch/sh/mm/pg-nommu.c @@ -14,12 +14,12 @@ #include #include -void copy_page_nommu(void *to, void *from) +void copy_page(void *to, void *from) { memcpy(to, from, PAGE_SIZE); } -void clear_page_nommu(void *to) +void clear_page(void *to) { memset(to, 0, PAGE_SIZE); } -- cgit v1.2.3 From 811d50cb43eb730cc325df0c6913556e25739797 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Tue, 20 Nov 2007 17:01:55 +0900 Subject: sh: Move in the SH-5 TLB miss. Signed-off-by: Paul Mundt --- arch/sh/mm/Makefile_32 | 2 +- arch/sh/mm/Makefile_64 | 2 +- arch/sh/mm/fault.c | 303 ------------------------------------------------- arch/sh/mm/fault_32.c | 303 +++++++++++++++++++++++++++++++++++++++++++++++++ arch/sh/mm/fault_64.c | 279 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 584 insertions(+), 305 deletions(-) delete mode 100644 arch/sh/mm/fault.c create mode 100644 arch/sh/mm/fault_32.c create mode 100644 arch/sh/mm/fault_64.c (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/Makefile_32 b/arch/sh/mm/Makefile_32 index 9ea379303742..fc089c0cdbe9 100644 --- a/arch/sh/mm/Makefile_32 +++ b/arch/sh/mm/Makefile_32 @@ -12,7 +12,7 @@ obj-$(CONFIG_SH7705_CACHE_32KB) += cache-sh7705.o endif mmu-y := tlb-nommu.o pg-nommu.o -mmu-$(CONFIG_MMU) := fault.o clear_page.o copy_page.o tlb-flush_32.o \ +mmu-$(CONFIG_MMU) := fault_32.o clear_page.o copy_page.o tlb-flush_32.o \ ioremap_32.o obj-y += $(mmu-y) diff --git a/arch/sh/mm/Makefile_64 b/arch/sh/mm/Makefile_64 index ec8deaa17909..82fe9072fd5c 100644 --- a/arch/sh/mm/Makefile_64 +++ b/arch/sh/mm/Makefile_64 @@ -5,7 +5,7 @@ obj-y := init.o extable_64.o consistent.o mmu-y := tlb-nommu.o pg-nommu.o -mmu-$(CONFIG_MMU) := ioremap_64.o tlb-flush_64.o +mmu-$(CONFIG_MMU) := fault_64.o ioremap_64.o tlb-flush_64.o obj-y += $(mmu-y) diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c deleted file mode 100644 index 60d74f793a1d..000000000000 --- a/arch/sh/mm/fault.c +++ /dev/null @@ -1,303 +0,0 @@ -/* - * Page fault handler for SH with an MMU. - * - * Copyright (C) 1999 Niibe Yutaka - * Copyright (C) 2003 - 2007 Paul Mundt - * - * Based on linux/arch/i386/mm/fault.c: - * Copyright (C) 1995 Linus Torvalds - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * This routine handles page faults. It determines the address, - * and the problem, and then passes it off to one of the appropriate - * routines. - */ -asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, - unsigned long writeaccess, - unsigned long address) -{ - struct task_struct *tsk; - struct mm_struct *mm; - struct vm_area_struct * vma; - int si_code; - int fault; - siginfo_t info; - - trace_hardirqs_on(); - local_irq_enable(); - -#ifdef CONFIG_SH_KGDB - if (kgdb_nofault && kgdb_bus_err_hook) - kgdb_bus_err_hook(); -#endif - - tsk = current; - mm = tsk->mm; - si_code = SEGV_MAPERR; - - if (unlikely(address >= TASK_SIZE)) { - /* - * Synchronize this task's top level page-table - * with the 'reference' page table. - * - * Do _not_ use "tsk" here. We might be inside - * an interrupt in the middle of a task switch.. - */ - int offset = pgd_index(address); - pgd_t *pgd, *pgd_k; - pud_t *pud, *pud_k; - pmd_t *pmd, *pmd_k; - - pgd = get_TTB() + offset; - pgd_k = swapper_pg_dir + offset; - - /* This will never happen with the folded page table. */ - if (!pgd_present(*pgd)) { - if (!pgd_present(*pgd_k)) - goto bad_area_nosemaphore; - set_pgd(pgd, *pgd_k); - return; - } - - pud = pud_offset(pgd, address); - pud_k = pud_offset(pgd_k, address); - if (pud_present(*pud) || !pud_present(*pud_k)) - goto bad_area_nosemaphore; - set_pud(pud, *pud_k); - - pmd = pmd_offset(pud, address); - pmd_k = pmd_offset(pud_k, address); - if (pmd_present(*pmd) || !pmd_present(*pmd_k)) - goto bad_area_nosemaphore; - set_pmd(pmd, *pmd_k); - - return; - } - - /* - * If we're in an interrupt or have no user - * context, we must not take the fault.. - */ - if (in_atomic() || !mm) - goto no_context; - - down_read(&mm->mmap_sem); - - vma = find_vma(mm, address); - if (!vma) - goto bad_area; - if (vma->vm_start <= address) - goto good_area; - if (!(vma->vm_flags & VM_GROWSDOWN)) - goto bad_area; - if (expand_stack(vma, address)) - goto bad_area; -/* - * Ok, we have a good vm_area for this memory access, so - * we can handle it.. - */ -good_area: - si_code = SEGV_ACCERR; - if (writeaccess) { - if (!(vma->vm_flags & VM_WRITE)) - goto bad_area; - } else { - if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) - goto bad_area; - } - - /* - * If for any reason at all we couldn't handle the fault, - * make sure we exit gracefully rather than endlessly redo - * the fault. - */ -survive: - fault = handle_mm_fault(mm, vma, address, writeaccess); - if (unlikely(fault & VM_FAULT_ERROR)) { - if (fault & VM_FAULT_OOM) - goto out_of_memory; - else if (fault & VM_FAULT_SIGBUS) - goto do_sigbus; - BUG(); - } - if (fault & VM_FAULT_MAJOR) - tsk->maj_flt++; - else - tsk->min_flt++; - - up_read(&mm->mmap_sem); - return; - -/* - * Something tried to access memory that isn't in our memory map.. - * Fix it, but check if it's kernel or user first.. - */ -bad_area: - up_read(&mm->mmap_sem); - -bad_area_nosemaphore: - if (user_mode(regs)) { - info.si_signo = SIGSEGV; - info.si_errno = 0; - info.si_code = si_code; - info.si_addr = (void *) address; - force_sig_info(SIGSEGV, &info, tsk); - return; - } - -no_context: - /* Are we prepared to handle this kernel fault? */ - if (fixup_exception(regs)) - return; - -/* - * Oops. The kernel tried to access some bad page. We'll have to - * terminate things with extreme prejudice. - * - */ - - bust_spinlocks(1); - - if (oops_may_print()) { - __typeof__(pte_val(__pte(0))) page; - - if (address < PAGE_SIZE) - printk(KERN_ALERT "Unable to handle kernel NULL " - "pointer dereference"); - else - printk(KERN_ALERT "Unable to handle kernel paging " - "request"); - printk(" at virtual address %08lx\n", address); - printk(KERN_ALERT "pc = %08lx\n", regs->pc); - page = (unsigned long)get_TTB(); - if (page) { - page = ((__typeof__(page) *)page)[address >> PGDIR_SHIFT]; - printk(KERN_ALERT "*pde = %08lx\n", page); - if (page & _PAGE_PRESENT) { - page &= PAGE_MASK; - address &= 0x003ff000; - page = ((__typeof__(page) *) - __va(page))[address >> - PAGE_SHIFT]; - printk(KERN_ALERT "*pte = %08lx\n", page); - } - } - } - - die("Oops", regs, writeaccess); - bust_spinlocks(0); - do_exit(SIGKILL); - -/* - * We ran out of memory, or some other thing happened to us that made - * us unable to handle the page fault gracefully. - */ -out_of_memory: - up_read(&mm->mmap_sem); - if (is_global_init(current)) { - yield(); - down_read(&mm->mmap_sem); - goto survive; - } - printk("VM: killing process %s\n", tsk->comm); - if (user_mode(regs)) - do_group_exit(SIGKILL); - goto no_context; - -do_sigbus: - up_read(&mm->mmap_sem); - - /* - * Send a sigbus, regardless of whether we were in kernel - * or user mode. - */ - info.si_signo = SIGBUS; - info.si_errno = 0; - info.si_code = BUS_ADRERR; - info.si_addr = (void *)address; - force_sig_info(SIGBUS, &info, tsk); - - /* Kernel mode? Handle exceptions or die */ - if (!user_mode(regs)) - goto no_context; -} - -#ifdef CONFIG_SH_STORE_QUEUES -/* - * This is a special case for the SH-4 store queues, as pages for this - * space still need to be faulted in before it's possible to flush the - * store queue cache for writeout to the remapped region. - */ -#define P3_ADDR_MAX (P4SEG_STORE_QUE + 0x04000000) -#else -#define P3_ADDR_MAX P4SEG -#endif - -/* - * Called with interrupts disabled. - */ -asmlinkage int __kprobes __do_page_fault(struct pt_regs *regs, - unsigned long writeaccess, - unsigned long address) -{ - pgd_t *pgd; - pud_t *pud; - pmd_t *pmd; - pte_t *pte; - pte_t entry; - -#ifdef CONFIG_SH_KGDB - if (kgdb_nofault && kgdb_bus_err_hook) - kgdb_bus_err_hook(); -#endif - - /* - * We don't take page faults for P1, P2, and parts of P4, these - * are always mapped, whether it be due to legacy behaviour in - * 29-bit mode, or due to PMB configuration in 32-bit mode. - */ - if (address >= P3SEG && address < P3_ADDR_MAX) { - pgd = pgd_offset_k(address); - } else { - if (unlikely(address >= TASK_SIZE || !current->mm)) - return 1; - - pgd = pgd_offset(current->mm, address); - } - - pud = pud_offset(pgd, address); - if (pud_none_or_clear_bad(pud)) - return 1; - pmd = pmd_offset(pud, address); - if (pmd_none_or_clear_bad(pmd)) - return 1; - - pte = pte_offset_kernel(pmd, address); - entry = *pte; - if (unlikely(pte_none(entry) || pte_not_present(entry))) - return 1; - if (unlikely(writeaccess && !pte_write(entry))) - return 1; - - if (writeaccess) - entry = pte_mkdirty(entry); - entry = pte_mkyoung(entry); - - set_pte(pte, entry); - update_mmu_cache(NULL, address, entry); - - return 0; -} diff --git a/arch/sh/mm/fault_32.c b/arch/sh/mm/fault_32.c new file mode 100644 index 000000000000..60d74f793a1d --- /dev/null +++ b/arch/sh/mm/fault_32.c @@ -0,0 +1,303 @@ +/* + * Page fault handler for SH with an MMU. + * + * Copyright (C) 1999 Niibe Yutaka + * Copyright (C) 2003 - 2007 Paul Mundt + * + * Based on linux/arch/i386/mm/fault.c: + * Copyright (C) 1995 Linus Torvalds + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * This routine handles page faults. It determines the address, + * and the problem, and then passes it off to one of the appropriate + * routines. + */ +asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, + unsigned long writeaccess, + unsigned long address) +{ + struct task_struct *tsk; + struct mm_struct *mm; + struct vm_area_struct * vma; + int si_code; + int fault; + siginfo_t info; + + trace_hardirqs_on(); + local_irq_enable(); + +#ifdef CONFIG_SH_KGDB + if (kgdb_nofault && kgdb_bus_err_hook) + kgdb_bus_err_hook(); +#endif + + tsk = current; + mm = tsk->mm; + si_code = SEGV_MAPERR; + + if (unlikely(address >= TASK_SIZE)) { + /* + * Synchronize this task's top level page-table + * with the 'reference' page table. + * + * Do _not_ use "tsk" here. We might be inside + * an interrupt in the middle of a task switch.. + */ + int offset = pgd_index(address); + pgd_t *pgd, *pgd_k; + pud_t *pud, *pud_k; + pmd_t *pmd, *pmd_k; + + pgd = get_TTB() + offset; + pgd_k = swapper_pg_dir + offset; + + /* This will never happen with the folded page table. */ + if (!pgd_present(*pgd)) { + if (!pgd_present(*pgd_k)) + goto bad_area_nosemaphore; + set_pgd(pgd, *pgd_k); + return; + } + + pud = pud_offset(pgd, address); + pud_k = pud_offset(pgd_k, address); + if (pud_present(*pud) || !pud_present(*pud_k)) + goto bad_area_nosemaphore; + set_pud(pud, *pud_k); + + pmd = pmd_offset(pud, address); + pmd_k = pmd_offset(pud_k, address); + if (pmd_present(*pmd) || !pmd_present(*pmd_k)) + goto bad_area_nosemaphore; + set_pmd(pmd, *pmd_k); + + return; + } + + /* + * If we're in an interrupt or have no user + * context, we must not take the fault.. + */ + if (in_atomic() || !mm) + goto no_context; + + down_read(&mm->mmap_sem); + + vma = find_vma(mm, address); + if (!vma) + goto bad_area; + if (vma->vm_start <= address) + goto good_area; + if (!(vma->vm_flags & VM_GROWSDOWN)) + goto bad_area; + if (expand_stack(vma, address)) + goto bad_area; +/* + * Ok, we have a good vm_area for this memory access, so + * we can handle it.. + */ +good_area: + si_code = SEGV_ACCERR; + if (writeaccess) { + if (!(vma->vm_flags & VM_WRITE)) + goto bad_area; + } else { + if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) + goto bad_area; + } + + /* + * If for any reason at all we couldn't handle the fault, + * make sure we exit gracefully rather than endlessly redo + * the fault. + */ +survive: + fault = handle_mm_fault(mm, vma, address, writeaccess); + if (unlikely(fault & VM_FAULT_ERROR)) { + if (fault & VM_FAULT_OOM) + goto out_of_memory; + else if (fault & VM_FAULT_SIGBUS) + goto do_sigbus; + BUG(); + } + if (fault & VM_FAULT_MAJOR) + tsk->maj_flt++; + else + tsk->min_flt++; + + up_read(&mm->mmap_sem); + return; + +/* + * Something tried to access memory that isn't in our memory map.. + * Fix it, but check if it's kernel or user first.. + */ +bad_area: + up_read(&mm->mmap_sem); + +bad_area_nosemaphore: + if (user_mode(regs)) { + info.si_signo = SIGSEGV; + info.si_errno = 0; + info.si_code = si_code; + info.si_addr = (void *) address; + force_sig_info(SIGSEGV, &info, tsk); + return; + } + +no_context: + /* Are we prepared to handle this kernel fault? */ + if (fixup_exception(regs)) + return; + +/* + * Oops. The kernel tried to access some bad page. We'll have to + * terminate things with extreme prejudice. + * + */ + + bust_spinlocks(1); + + if (oops_may_print()) { + __typeof__(pte_val(__pte(0))) page; + + if (address < PAGE_SIZE) + printk(KERN_ALERT "Unable to handle kernel NULL " + "pointer dereference"); + else + printk(KERN_ALERT "Unable to handle kernel paging " + "request"); + printk(" at virtual address %08lx\n", address); + printk(KERN_ALERT "pc = %08lx\n", regs->pc); + page = (unsigned long)get_TTB(); + if (page) { + page = ((__typeof__(page) *)page)[address >> PGDIR_SHIFT]; + printk(KERN_ALERT "*pde = %08lx\n", page); + if (page & _PAGE_PRESENT) { + page &= PAGE_MASK; + address &= 0x003ff000; + page = ((__typeof__(page) *) + __va(page))[address >> + PAGE_SHIFT]; + printk(KERN_ALERT "*pte = %08lx\n", page); + } + } + } + + die("Oops", regs, writeaccess); + bust_spinlocks(0); + do_exit(SIGKILL); + +/* + * We ran out of memory, or some other thing happened to us that made + * us unable to handle the page fault gracefully. + */ +out_of_memory: + up_read(&mm->mmap_sem); + if (is_global_init(current)) { + yield(); + down_read(&mm->mmap_sem); + goto survive; + } + printk("VM: killing process %s\n", tsk->comm); + if (user_mode(regs)) + do_group_exit(SIGKILL); + goto no_context; + +do_sigbus: + up_read(&mm->mmap_sem); + + /* + * Send a sigbus, regardless of whether we were in kernel + * or user mode. + */ + info.si_signo = SIGBUS; + info.si_errno = 0; + info.si_code = BUS_ADRERR; + info.si_addr = (void *)address; + force_sig_info(SIGBUS, &info, tsk); + + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) + goto no_context; +} + +#ifdef CONFIG_SH_STORE_QUEUES +/* + * This is a special case for the SH-4 store queues, as pages for this + * space still need to be faulted in before it's possible to flush the + * store queue cache for writeout to the remapped region. + */ +#define P3_ADDR_MAX (P4SEG_STORE_QUE + 0x04000000) +#else +#define P3_ADDR_MAX P4SEG +#endif + +/* + * Called with interrupts disabled. + */ +asmlinkage int __kprobes __do_page_fault(struct pt_regs *regs, + unsigned long writeaccess, + unsigned long address) +{ + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + pte_t entry; + +#ifdef CONFIG_SH_KGDB + if (kgdb_nofault && kgdb_bus_err_hook) + kgdb_bus_err_hook(); +#endif + + /* + * We don't take page faults for P1, P2, and parts of P4, these + * are always mapped, whether it be due to legacy behaviour in + * 29-bit mode, or due to PMB configuration in 32-bit mode. + */ + if (address >= P3SEG && address < P3_ADDR_MAX) { + pgd = pgd_offset_k(address); + } else { + if (unlikely(address >= TASK_SIZE || !current->mm)) + return 1; + + pgd = pgd_offset(current->mm, address); + } + + pud = pud_offset(pgd, address); + if (pud_none_or_clear_bad(pud)) + return 1; + pmd = pmd_offset(pud, address); + if (pmd_none_or_clear_bad(pmd)) + return 1; + + pte = pte_offset_kernel(pmd, address); + entry = *pte; + if (unlikely(pte_none(entry) || pte_not_present(entry))) + return 1; + if (unlikely(writeaccess && !pte_write(entry))) + return 1; + + if (writeaccess) + entry = pte_mkdirty(entry); + entry = pte_mkyoung(entry); + + set_pte(pte, entry); + update_mmu_cache(NULL, address, entry); + + return 0; +} diff --git a/arch/sh/mm/fault_64.c b/arch/sh/mm/fault_64.c new file mode 100644 index 000000000000..1469aa70bc78 --- /dev/null +++ b/arch/sh/mm/fault_64.c @@ -0,0 +1,279 @@ +/* + * The SH64 TLB miss. + * + * Original code from fault.c + * Copyright (C) 2000, 2001 Paolo Alberelli + * + * Fast PTE->TLB refill path + * Copyright (C) 2003 Richard.Curnow@superh.com + * + * IMPORTANT NOTES : + * The do_fast_page_fault function is called from a context in entry.S + * where very few registers have been saved. In particular, the code in + * this file must be compiled not to use ANY caller-save registers that + * are not part of the restricted save set. Also, it means that code in + * this file must not make calls to functions elsewhere in the kernel, or + * else the excepting context will see corruption in its caller-save + * registers. Plus, the entry.S save area is non-reentrant, so this code + * has to run with SR.BL==1, i.e. no interrupts taken inside it and panic + * on any exception. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Callable from fault.c, so not static */ +inline void __do_tlb_refill(unsigned long address, + unsigned long long is_text_not_data, pte_t *pte) +{ + unsigned long long ptel; + unsigned long long pteh=0; + struct tlb_info *tlbp; + unsigned long long next; + + /* Get PTEL first */ + ptel = pte_val(*pte); + + /* + * Set PTEH register + */ + pteh = address & MMU_VPN_MASK; + + /* Sign extend based on neff. */ +#if (NEFF == 32) + /* Faster sign extension */ + pteh = (unsigned long long)(signed long long)(signed long)pteh; +#else + /* General case */ + pteh = (pteh & NEFF_SIGN) ? (pteh | NEFF_MASK) : pteh; +#endif + + /* Set the ASID. */ + pteh |= get_asid() << PTEH_ASID_SHIFT; + pteh |= PTEH_VALID; + + /* Set PTEL register, set_pte has performed the sign extension */ + ptel &= _PAGE_FLAGS_HARDWARE_MASK; /* drop software flags */ + + tlbp = is_text_not_data ? &(cpu_data->itlb) : &(cpu_data->dtlb); + next = tlbp->next; + __flush_tlb_slot(next); + asm volatile ("putcfg %0,1,%2\n\n\t" + "putcfg %0,0,%1\n" + : : "r" (next), "r" (pteh), "r" (ptel) ); + + next += TLB_STEP; + if (next > tlbp->last) next = tlbp->first; + tlbp->next = next; + +} + +static int handle_vmalloc_fault(struct mm_struct *mm, + unsigned long protection_flags, + unsigned long long textaccess, + unsigned long address) +{ + pgd_t *dir; + pud_t *pud; + pmd_t *pmd; + static pte_t *pte; + pte_t entry; + + dir = pgd_offset_k(address); + + pud = pud_offset(dir, address); + if (pud_none_or_clear_bad(pud)) + return 0; + + pmd = pmd_offset(pud, address); + if (pmd_none_or_clear_bad(pmd)) + return 0; + + pte = pte_offset_kernel(pmd, address); + entry = *pte; + + if (pte_none(entry) || !pte_present(entry)) + return 0; + if ((pte_val(entry) & protection_flags) != protection_flags) + return 0; + + __do_tlb_refill(address, textaccess, pte); + + return 1; +} + +static int handle_tlbmiss(struct mm_struct *mm, + unsigned long long protection_flags, + unsigned long long textaccess, + unsigned long address) +{ + pgd_t *dir; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + pte_t entry; + + /* NB. The PGD currently only contains a single entry - there is no + page table tree stored for the top half of the address space since + virtual pages in that region should never be mapped in user mode. + (In kernel mode, the only things in that region are the 512Mb super + page (locked in), and vmalloc (modules) + I/O device pages (handled + by handle_vmalloc_fault), so no PGD for the upper half is required + by kernel mode either). + + See how mm->pgd is allocated and initialised in pgd_alloc to see why + the next test is necessary. - RPC */ + if (address >= (unsigned long) TASK_SIZE) + /* upper half - never has page table entries. */ + return 0; + + dir = pgd_offset(mm, address); + if (pgd_none(*dir) || !pgd_present(*dir)) + return 0; + if (!pgd_present(*dir)) + return 0; + + pud = pud_offset(dir, address); + if (pud_none(*pud) || !pud_present(*pud)) + return 0; + + pmd = pmd_offset(pud, address); + if (pmd_none(*pmd) || !pmd_present(*pmd)) + return 0; + + pte = pte_offset_kernel(pmd, address); + entry = *pte; + + if (pte_none(entry) || !pte_present(entry)) + return 0; + + /* + * If the page doesn't have sufficient protection bits set to + * service the kind of fault being handled, there's not much + * point doing the TLB refill. Punt the fault to the general + * handler. + */ + if ((pte_val(entry) & protection_flags) != protection_flags) + return 0; + + __do_tlb_refill(address, textaccess, pte); + + return 1; +} + +/* + * Put all this information into one structure so that everything is just + * arithmetic relative to a single base address. This reduces the number + * of movi/shori pairs needed just to load addresses of static data. + */ +struct expevt_lookup { + unsigned short protection_flags[8]; + unsigned char is_text_access[8]; + unsigned char is_write_access[8]; +}; + +#define PRU (1<<9) +#define PRW (1<<8) +#define PRX (1<<7) +#define PRR (1<<6) + +#define DIRTY (_PAGE_DIRTY | _PAGE_ACCESSED) +#define YOUNG (_PAGE_ACCESSED) + +/* Sized as 8 rather than 4 to allow checking the PTE's PRU bit against whether + the fault happened in user mode or privileged mode. */ +static struct expevt_lookup expevt_lookup_table = { + .protection_flags = {PRX, PRX, 0, 0, PRR, PRR, PRW, PRW}, + .is_text_access = {1, 1, 0, 0, 0, 0, 0, 0} +}; + +/* + This routine handles page faults that can be serviced just by refilling a + TLB entry from an existing page table entry. (This case represents a very + large majority of page faults.) Return 1 if the fault was successfully + handled. Return 0 if the fault could not be handled. (This leads into the + general fault handling in fault.c which deals with mapping file-backed + pages, stack growth, segmentation faults, swapping etc etc) + */ +asmlinkage int do_fast_page_fault(unsigned long long ssr_md, + unsigned long long expevt, + unsigned long address) +{ + struct task_struct *tsk; + struct mm_struct *mm; + unsigned long long textaccess; + unsigned long long protection_flags; + unsigned long long index; + unsigned long long expevt4; + + /* The next few lines implement a way of hashing EXPEVT into a + * small array index which can be used to lookup parameters + * specific to the type of TLBMISS being handled. + * + * Note: + * ITLBMISS has EXPEVT==0xa40 + * RTLBMISS has EXPEVT==0x040 + * WTLBMISS has EXPEVT==0x060 + */ + expevt4 = (expevt >> 4); + /* TODO : xor ssr_md into this expression too. Then we can check + * that PRU is set when it needs to be. */ + index = expevt4 ^ (expevt4 >> 5); + index &= 7; + protection_flags = expevt_lookup_table.protection_flags[index]; + textaccess = expevt_lookup_table.is_text_access[index]; + +#ifdef CONFIG_SH64_PROC_TLB + ++calls_to_do_fast_page_fault; +#endif + + /* SIM + * Note this is now called with interrupts still disabled + * This is to cope with being called for a missing IO port + * address with interrupts disabled. This should be fixed as + * soon as we have a better 'fast path' miss handler. + * + * Plus take care how you try and debug this stuff. + * For example, writing debug data to a port which you + * have just faulted on is not going to work. + */ + + tsk = current; + mm = tsk->mm; + + if ((address >= VMALLOC_START && address < VMALLOC_END) || + (address >= IOBASE_VADDR && address < IOBASE_END)) { + if (ssr_md) + /* + * Process-contexts can never have this address + * range mapped + */ + if (handle_vmalloc_fault(mm, protection_flags, + textaccess, address)) + return 1; + } else if (!in_interrupt() && mm) { + if (handle_tlbmiss(mm, protection_flags, textaccess, address)) + return 1; + } + + return 0; +} -- cgit v1.2.3 From 36763b22bee56713057762bb0756e8075b5bb918 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Wed, 21 Nov 2007 15:34:33 +0900 Subject: sh: Switch SH-5 to use CONFIG_PAGE_OFFSET. Signed-off-by: Paul Mundt --- arch/sh/mm/Kconfig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/Kconfig b/arch/sh/mm/Kconfig index 7ff26c1f4e9c..2af950784e66 100644 --- a/arch/sh/mm/Kconfig +++ b/arch/sh/mm/Kconfig @@ -17,7 +17,8 @@ config MMU config PAGE_OFFSET hex - default "0x80000000" if MMU + default "0x80000000" if MMU && SUPERH32 + default "0x20000000" if MMU && SUPERH64 default "0x00000000" config MEMORY_START -- cgit v1.2.3 From 711fe4369c5d3834cfe1750f8804ef2b74865e9e Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Wed, 21 Nov 2007 15:46:07 +0900 Subject: sh: Bump the MEMORY_SIZE default to something reasonable. Signed-off-by: Paul Mundt --- arch/sh/mm/Kconfig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/Kconfig b/arch/sh/mm/Kconfig index 2af950784e66..f549b8cd2501 100644 --- a/arch/sh/mm/Kconfig +++ b/arch/sh/mm/Kconfig @@ -39,12 +39,12 @@ config MEMORY_START config MEMORY_SIZE hex "Physical memory size" - default "0x00400000" + default "0x04000000" help This sets the default memory size assumed by your SH kernel. It can be overridden as normal by the 'mem=' argument on the kernel command line. If unsure, consult your board specifications or just leave it - as 0x00400000 which was the default value before this became + as 0x04000000 which was the default value before this became configurable. # Physical addressing modes -- cgit v1.2.3 From ddf7f1d339861b9867342edccc9c6b864ca9dd26 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Wed, 21 Nov 2007 16:55:27 +0900 Subject: sh: Kill off a stray proc tlb reference in fault_64. Signed-off-by: Paul Mundt --- arch/sh/mm/fault_64.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/fault_64.c b/arch/sh/mm/fault_64.c index 1469aa70bc78..399d53710d2f 100644 --- a/arch/sh/mm/fault_64.c +++ b/arch/sh/mm/fault_64.c @@ -242,10 +242,6 @@ asmlinkage int do_fast_page_fault(unsigned long long ssr_md, protection_flags = expevt_lookup_table.protection_flags[index]; textaccess = expevt_lookup_table.is_text_access[index]; -#ifdef CONFIG_SH64_PROC_TLB - ++calls_to_do_fast_page_fault; -#endif - /* SIM * Note this is now called with interrupts still disabled * This is to cope with being called for a missing IO port -- cgit v1.2.3 From 60b2249d45d44bd3494d55f5ea4bccd25c7f8281 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Wed, 21 Nov 2007 17:07:06 +0900 Subject: sh: Move over SH-5 TLB and cache support code. Signed-off-by: Paul Mundt --- arch/sh/mm/Makefile_64 | 7 +- arch/sh/mm/cache-sh5.c | 1032 ++++++++++++++++++++++++++++++++++++++++++++++++ arch/sh/mm/tlb-sh5.c | 166 ++++++++ 3 files changed, 1203 insertions(+), 2 deletions(-) create mode 100644 arch/sh/mm/cache-sh5.c create mode 100644 arch/sh/mm/tlb-sh5.c (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/Makefile_64 b/arch/sh/mm/Makefile_64 index 82fe9072fd5c..5ff83583593e 100644 --- a/arch/sh/mm/Makefile_64 +++ b/arch/sh/mm/Makefile_64 @@ -5,12 +5,15 @@ obj-y := init.o extable_64.o consistent.o mmu-y := tlb-nommu.o pg-nommu.o -mmu-$(CONFIG_MMU) := fault_64.o ioremap_64.o tlb-flush_64.o +mmu-$(CONFIG_MMU) := fault_64.o ioremap_64.o tlb-flush_64.o tlb-sh5.o + +ifndef CONFIG_CACHE_OFF +obj-y += cache-sh5.o +endif obj-y += $(mmu-y) obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o -obj-$(CONFIG_PMB) += pmb.o obj-$(CONFIG_NUMA) += numa.o EXTRA_CFLAGS += -Werror diff --git a/arch/sh/mm/cache-sh5.c b/arch/sh/mm/cache-sh5.c new file mode 100644 index 000000000000..421487cfff4c --- /dev/null +++ b/arch/sh/mm/cache-sh5.c @@ -0,0 +1,1032 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * arch/sh64/mm/cache.c + * + * Original version Copyright (C) 2000, 2001 Paolo Alberelli + * Second version Copyright (C) benedict.gaster@superh.com 2002 + * Third version Copyright Richard.Curnow@superh.com 2003 + * Hacks to third version Copyright (C) 2003 Paul Mundt + */ + +/****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* for flush_itlb_range */ + +#include + +/* This function is in entry.S */ +extern unsigned long switch_and_save_asid(unsigned long new_asid); + +/* Wired TLB entry for the D-cache */ +static unsigned long long dtlb_cache_slot; + +/** + * sh64_cache_init() + * + * This is pretty much just a straightforward clone of the SH + * detect_cpu_and_cache_system(). + * + * This function is responsible for setting up all of the cache + * info dynamically as well as taking care of CPU probing and + * setting up the relevant subtype data. + * + * FIXME: For the time being, we only really support the SH5-101 + * out of the box, and don't support dynamic probing for things + * like the SH5-103 or even cut2 of the SH5-101. Implement this + * later! + */ +int __init sh64_cache_init(void) +{ + /* + * First, setup some sane values for the I-cache. + */ + cpu_data->icache.ways = 4; + cpu_data->icache.sets = 256; + cpu_data->icache.linesz = L1_CACHE_BYTES; + + /* + * FIXME: This can probably be cleaned up a bit as well.. for example, + * do we really need the way shift _and_ the way_step_shift ?? Judging + * by the existing code, I would guess no.. is there any valid reason + * why we need to be tracking this around? + */ + cpu_data->icache.way_shift = 13; + cpu_data->icache.entry_shift = 5; + cpu_data->icache.set_shift = 4; + cpu_data->icache.way_step_shift = 16; + cpu_data->icache.asid_shift = 2; + + /* + * way offset = cache size / associativity, so just don't factor in + * associativity in the first place.. + */ + cpu_data->icache.way_ofs = cpu_data->icache.sets * + cpu_data->icache.linesz; + + cpu_data->icache.asid_mask = 0x3fc; + cpu_data->icache.idx_mask = 0x1fe0; + cpu_data->icache.epn_mask = 0xffffe000; + cpu_data->icache.flags = 0; + + /* + * Next, setup some sane values for the D-cache. + * + * On the SH5, these are pretty consistent with the I-cache settings, + * so we just copy over the existing definitions.. these can be fixed + * up later, especially if we add runtime CPU probing. + * + * Though in the meantime it saves us from having to duplicate all of + * the above definitions.. + */ + cpu_data->dcache = cpu_data->icache; + + /* + * Setup any cache-related flags here + */ +#if defined(CONFIG_DCACHE_WRITE_THROUGH) + set_bit(SH_CACHE_MODE_WT, &(cpu_data->dcache.flags)); +#elif defined(CONFIG_DCACHE_WRITE_BACK) + set_bit(SH_CACHE_MODE_WB, &(cpu_data->dcache.flags)); +#endif + + /* + * We also need to reserve a slot for the D-cache in the DTLB, so we + * do this now .. + */ + dtlb_cache_slot = sh64_get_wired_dtlb_entry(); + + return 0; +} + +#ifdef CONFIG_DCACHE_DISABLED +#define sh64_dcache_purge_all() do { } while (0) +#define sh64_dcache_purge_coloured_phy_page(paddr, eaddr) do { } while (0) +#define sh64_dcache_purge_user_range(mm, start, end) do { } while (0) +#define sh64_dcache_purge_phy_page(paddr) do { } while (0) +#define sh64_dcache_purge_virt_page(mm, eaddr) do { } while (0) +#define sh64_dcache_purge_kernel_range(start, end) do { } while (0) +#define sh64_dcache_wback_current_user_range(start, end) do { } while (0) +#endif + +/*##########################################################################*/ + +/* From here onwards, a rewrite of the implementation, + by Richard.Curnow@superh.com. + + The major changes in this compared to the old version are; + 1. use more selective purging through OCBP instead of using ALLOCO to purge + by natural replacement. This avoids purging out unrelated cache lines + that happen to be in the same set. + 2. exploit the APIs copy_user_page and clear_user_page better + 3. be more selective about I-cache purging, in particular use invalidate_all + more sparingly. + + */ + +/*########################################################################## + SUPPORT FUNCTIONS + ##########################################################################*/ + +/****************************************************************************/ +/* The following group of functions deal with mapping and unmapping a temporary + page into the DTLB slot that have been set aside for our exclusive use. */ +/* In order to accomplish this, we use the generic interface for adding and + removing a wired slot entry as defined in arch/sh64/mm/tlb.c */ +/****************************************************************************/ + +static unsigned long slot_own_flags; + +static inline void sh64_setup_dtlb_cache_slot(unsigned long eaddr, unsigned long asid, unsigned long paddr) +{ + local_irq_save(slot_own_flags); + sh64_setup_tlb_slot(dtlb_cache_slot, eaddr, asid, paddr); +} + +static inline void sh64_teardown_dtlb_cache_slot(void) +{ + sh64_teardown_tlb_slot(dtlb_cache_slot); + local_irq_restore(slot_own_flags); +} + +/****************************************************************************/ + +#ifndef CONFIG_ICACHE_DISABLED + +static void __inline__ sh64_icache_inv_all(void) +{ + unsigned long long addr, flag, data; + unsigned int flags; + + addr=ICCR0; + flag=ICCR0_ICI; + data=0; + + /* Make this a critical section for safety (probably not strictly necessary.) */ + local_irq_save(flags); + + /* Without %1 it gets unexplicably wrong */ + asm volatile("getcfg %3, 0, %0\n\t" + "or %0, %2, %0\n\t" + "putcfg %3, 0, %0\n\t" + "synci" + : "=&r" (data) + : "0" (data), "r" (flag), "r" (addr)); + + local_irq_restore(flags); +} + +static void sh64_icache_inv_kernel_range(unsigned long start, unsigned long end) +{ + /* Invalidate range of addresses [start,end] from the I-cache, where + * the addresses lie in the kernel superpage. */ + + unsigned long long ullend, addr, aligned_start; +#if (NEFF == 32) + aligned_start = (unsigned long long)(signed long long)(signed long) start; +#else +#error "NEFF != 32" +#endif + aligned_start &= L1_CACHE_ALIGN_MASK; + addr = aligned_start; +#if (NEFF == 32) + ullend = (unsigned long long) (signed long long) (signed long) end; +#else +#error "NEFF != 32" +#endif + while (addr <= ullend) { + asm __volatile__ ("icbi %0, 0" : : "r" (addr)); + addr += L1_CACHE_BYTES; + } +} + +static void sh64_icache_inv_user_page(struct vm_area_struct *vma, unsigned long eaddr) +{ + /* If we get called, we know that vma->vm_flags contains VM_EXEC. + Also, eaddr is page-aligned. */ + + unsigned long long addr, end_addr; + unsigned long flags = 0; + unsigned long running_asid, vma_asid; + addr = eaddr; + end_addr = addr + PAGE_SIZE; + + /* Check whether we can use the current ASID for the I-cache + invalidation. For example, if we're called via + access_process_vm->flush_cache_page->here, (e.g. when reading from + /proc), 'running_asid' will be that of the reader, not of the + victim. + + Also, note the risk that we might get pre-empted between the ASID + compare and blocking IRQs, and before we regain control, the + pid->ASID mapping changes. However, the whole cache will get + invalidated when the mapping is renewed, so the worst that can + happen is that the loop below ends up invalidating somebody else's + cache entries. + */ + + running_asid = get_asid(); + vma_asid = (vma->vm_mm->context & MMU_CONTEXT_ASID_MASK); + if (running_asid != vma_asid) { + local_irq_save(flags); + switch_and_save_asid(vma_asid); + } + while (addr < end_addr) { + /* Worth unrolling a little */ + asm __volatile__("icbi %0, 0" : : "r" (addr)); + asm __volatile__("icbi %0, 32" : : "r" (addr)); + asm __volatile__("icbi %0, 64" : : "r" (addr)); + asm __volatile__("icbi %0, 96" : : "r" (addr)); + addr += 128; + } + if (running_asid != vma_asid) { + switch_and_save_asid(running_asid); + local_irq_restore(flags); + } +} + +/****************************************************************************/ + +static void sh64_icache_inv_user_page_range(struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + /* Used for invalidating big chunks of I-cache, i.e. assume the range + is whole pages. If 'start' or 'end' is not page aligned, the code + is conservative and invalidates to the ends of the enclosing pages. + This is functionally OK, just a performance loss. */ + + /* See the comments below in sh64_dcache_purge_user_range() regarding + the choice of algorithm. However, for the I-cache option (2) isn't + available because there are no physical tags so aliases can't be + resolved. The icbi instruction has to be used through the user + mapping. Because icbi is cheaper than ocbp on a cache hit, it + would be cheaper to use the selective code for a large range than is + possible with the D-cache. Just assume 64 for now as a working + figure. + */ + + int n_pages; + + if (!mm) return; + + n_pages = ((end - start) >> PAGE_SHIFT); + if (n_pages >= 64) { + sh64_icache_inv_all(); + } else { + unsigned long aligned_start; + unsigned long eaddr; + unsigned long after_last_page_start; + unsigned long mm_asid, current_asid; + unsigned long long flags = 0ULL; + + mm_asid = mm->context & MMU_CONTEXT_ASID_MASK; + current_asid = get_asid(); + + if (mm_asid != current_asid) { + /* Switch ASID and run the invalidate loop under cli */ + local_irq_save(flags); + switch_and_save_asid(mm_asid); + } + + aligned_start = start & PAGE_MASK; + after_last_page_start = PAGE_SIZE + ((end - 1) & PAGE_MASK); + + while (aligned_start < after_last_page_start) { + struct vm_area_struct *vma; + unsigned long vma_end; + vma = find_vma(mm, aligned_start); + if (!vma || (aligned_start <= vma->vm_end)) { + /* Avoid getting stuck in an error condition */ + aligned_start += PAGE_SIZE; + continue; + } + vma_end = vma->vm_end; + if (vma->vm_flags & VM_EXEC) { + /* Executable */ + eaddr = aligned_start; + while (eaddr < vma_end) { + sh64_icache_inv_user_page(vma, eaddr); + eaddr += PAGE_SIZE; + } + } + aligned_start = vma->vm_end; /* Skip to start of next region */ + } + if (mm_asid != current_asid) { + switch_and_save_asid(current_asid); + local_irq_restore(flags); + } + } +} + +static void sh64_icache_inv_user_small_range(struct mm_struct *mm, + unsigned long start, int len) +{ + + /* Invalidate a small range of user context I-cache, not necessarily + page (or even cache-line) aligned. */ + + unsigned long long eaddr = start; + unsigned long long eaddr_end = start + len; + unsigned long current_asid, mm_asid; + unsigned long long flags; + unsigned long long epage_start; + + /* Since this is used inside ptrace, the ASID in the mm context + typically won't match current_asid. We'll have to switch ASID to do + this. For safety, and given that the range will be small, do all + this under cli. + + Note, there is a hazard that the ASID in mm->context is no longer + actually associated with mm, i.e. if the mm->context has started a + new cycle since mm was last active. However, this is just a + performance issue: all that happens is that we invalidate lines + belonging to another mm, so the owning process has to refill them + when that mm goes live again. mm itself can't have any cache + entries because there will have been a flush_cache_all when the new + mm->context cycle started. */ + + /* Align to start of cache line. Otherwise, suppose len==8 and start + was at 32N+28 : the last 4 bytes wouldn't get invalidated. */ + eaddr = start & L1_CACHE_ALIGN_MASK; + eaddr_end = start + len; + + local_irq_save(flags); + mm_asid = mm->context & MMU_CONTEXT_ASID_MASK; + current_asid = switch_and_save_asid(mm_asid); + + epage_start = eaddr & PAGE_MASK; + + while (eaddr < eaddr_end) + { + asm __volatile__("icbi %0, 0" : : "r" (eaddr)); + eaddr += L1_CACHE_BYTES; + } + switch_and_save_asid(current_asid); + local_irq_restore(flags); +} + +static void sh64_icache_inv_current_user_range(unsigned long start, unsigned long end) +{ + /* The icbi instruction never raises ITLBMISS. i.e. if there's not a + cache hit on the virtual tag the instruction ends there, without a + TLB lookup. */ + + unsigned long long aligned_start; + unsigned long long ull_end; + unsigned long long addr; + + ull_end = end; + + /* Just invalidate over the range using the natural addresses. TLB + miss handling will be OK (TBC). Since it's for the current process, + either we're already in the right ASID context, or the ASIDs have + been recycled since we were last active in which case we might just + invalidate another processes I-cache entries : no worries, just a + performance drop for him. */ + aligned_start = start & L1_CACHE_ALIGN_MASK; + addr = aligned_start; + while (addr < ull_end) { + asm __volatile__ ("icbi %0, 0" : : "r" (addr)); + asm __volatile__ ("nop"); + asm __volatile__ ("nop"); + addr += L1_CACHE_BYTES; + } +} + +#endif /* !CONFIG_ICACHE_DISABLED */ + +/****************************************************************************/ + +#ifndef CONFIG_DCACHE_DISABLED + +/* Buffer used as the target of alloco instructions to purge data from cache + sets by natural eviction. -- RPC */ +#define DUMMY_ALLOCO_AREA_SIZE L1_CACHE_SIZE_BYTES + (1024 * 4) +static unsigned char dummy_alloco_area[DUMMY_ALLOCO_AREA_SIZE] __cacheline_aligned = { 0, }; + +/****************************************************************************/ + +static void __inline__ sh64_dcache_purge_sets(int sets_to_purge_base, int n_sets) +{ + /* Purge all ways in a particular block of sets, specified by the base + set number and number of sets. Can handle wrap-around, if that's + needed. */ + + int dummy_buffer_base_set; + unsigned long long eaddr, eaddr0, eaddr1; + int j; + int set_offset; + + dummy_buffer_base_set = ((int)&dummy_alloco_area & cpu_data->dcache.idx_mask) >> cpu_data->dcache.entry_shift; + set_offset = sets_to_purge_base - dummy_buffer_base_set; + + for (j=0; jdcache.sets - 1); + eaddr0 = (unsigned long long)dummy_alloco_area + (set_offset << cpu_data->dcache.entry_shift); + + /* Do one alloco which hits the required set per cache way. For + write-back mode, this will purge the #ways resident lines. There's + little point unrolling this loop because the allocos stall more if + they're too close together. */ + eaddr1 = eaddr0 + cpu_data->dcache.way_ofs * cpu_data->dcache.ways; + for (eaddr=eaddr0; eaddrdcache.way_ofs) { + asm __volatile__ ("alloco %0, 0" : : "r" (eaddr)); + asm __volatile__ ("synco"); /* TAKum03020 */ + } + + eaddr1 = eaddr0 + cpu_data->dcache.way_ofs * cpu_data->dcache.ways; + for (eaddr=eaddr0; eaddrdcache.way_ofs) { + /* Load from each address. Required because alloco is a NOP if + the cache is write-through. Write-through is a config option. */ + if (test_bit(SH_CACHE_MODE_WT, &(cpu_data->dcache.flags))) + *(volatile unsigned char *)(int)eaddr; + } + } + + /* Don't use OCBI to invalidate the lines. That costs cycles directly. + If the dummy block is just left resident, it will naturally get + evicted as required. */ + + return; +} + +/****************************************************************************/ + +static void sh64_dcache_purge_all(void) +{ + /* Purge the entire contents of the dcache. The most efficient way to + achieve this is to use alloco instructions on a region of unused + memory equal in size to the cache, thereby causing the current + contents to be discarded by natural eviction. The alternative, + namely reading every tag, setting up a mapping for the corresponding + page and doing an OCBP for the line, would be much more expensive. + */ + + sh64_dcache_purge_sets(0, cpu_data->dcache.sets); + + return; + +} + +/****************************************************************************/ + +static void sh64_dcache_purge_kernel_range(unsigned long start, unsigned long end) +{ + /* Purge the range of addresses [start,end] from the D-cache. The + addresses lie in the superpage mapping. There's no harm if we + overpurge at either end - just a small performance loss. */ + unsigned long long ullend, addr, aligned_start; +#if (NEFF == 32) + aligned_start = (unsigned long long)(signed long long)(signed long) start; +#else +#error "NEFF != 32" +#endif + aligned_start &= L1_CACHE_ALIGN_MASK; + addr = aligned_start; +#if (NEFF == 32) + ullend = (unsigned long long) (signed long long) (signed long) end; +#else +#error "NEFF != 32" +#endif + while (addr <= ullend) { + asm __volatile__ ("ocbp %0, 0" : : "r" (addr)); + addr += L1_CACHE_BYTES; + } + return; +} + +/* Assumes this address (+ (2**n_synbits) pages up from it) aren't used for + anything else in the kernel */ +#define MAGIC_PAGE0_START 0xffffffffec000000ULL + +static void sh64_dcache_purge_coloured_phy_page(unsigned long paddr, unsigned long eaddr) +{ + /* Purge the physical page 'paddr' from the cache. It's known that any + cache lines requiring attention have the same page colour as the the + address 'eaddr'. + + This relies on the fact that the D-cache matches on physical tags + when no virtual tag matches. So we create an alias for the original + page and purge through that. (Alternatively, we could have done + this by switching ASID to match the original mapping and purged + through that, but that involves ASID switching cost + probably a + TLBMISS + refill anyway.) + */ + + unsigned long long magic_page_start; + unsigned long long magic_eaddr, magic_eaddr_end; + + magic_page_start = MAGIC_PAGE0_START + (eaddr & CACHE_OC_SYN_MASK); + + /* As long as the kernel is not pre-emptible, this doesn't need to be + under cli/sti. */ + + sh64_setup_dtlb_cache_slot(magic_page_start, get_asid(), paddr); + + magic_eaddr = magic_page_start; + magic_eaddr_end = magic_eaddr + PAGE_SIZE; + while (magic_eaddr < magic_eaddr_end) { + /* Little point in unrolling this loop - the OCBPs are blocking + and won't go any quicker (i.e. the loop overhead is parallel + to part of the OCBP execution.) */ + asm __volatile__ ("ocbp %0, 0" : : "r" (magic_eaddr)); + magic_eaddr += L1_CACHE_BYTES; + } + + sh64_teardown_dtlb_cache_slot(); +} + +/****************************************************************************/ + +static void sh64_dcache_purge_phy_page(unsigned long paddr) +{ + /* Pure a page given its physical start address, by creating a + temporary 1 page mapping and purging across that. Even if we know + the virtual address (& vma or mm) of the page, the method here is + more elegant because it avoids issues of coping with page faults on + the purge instructions (i.e. no special-case code required in the + critical path in the TLB miss handling). */ + + unsigned long long eaddr_start, eaddr, eaddr_end; + int i; + + /* As long as the kernel is not pre-emptible, this doesn't need to be + under cli/sti. */ + + eaddr_start = MAGIC_PAGE0_START; + for (i=0; i < (1 << CACHE_OC_N_SYNBITS); i++) { + sh64_setup_dtlb_cache_slot(eaddr_start, get_asid(), paddr); + + eaddr = eaddr_start; + eaddr_end = eaddr + PAGE_SIZE; + while (eaddr < eaddr_end) { + asm __volatile__ ("ocbp %0, 0" : : "r" (eaddr)); + eaddr += L1_CACHE_BYTES; + } + + sh64_teardown_dtlb_cache_slot(); + eaddr_start += PAGE_SIZE; + } +} + +static void sh64_dcache_purge_user_pages(struct mm_struct *mm, + unsigned long addr, unsigned long end) +{ + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + pte_t entry; + spinlock_t *ptl; + unsigned long paddr; + + if (!mm) + return; /* No way to find physical address of page */ + + pgd = pgd_offset(mm, addr); + if (pgd_bad(*pgd)) + return; + + pmd = pmd_offset(pgd, addr); + if (pmd_none(*pmd) || pmd_bad(*pmd)) + return; + + pte = pte_offset_map_lock(mm, pmd, addr, &ptl); + do { + entry = *pte; + if (pte_none(entry) || !pte_present(entry)) + continue; + paddr = pte_val(entry) & PAGE_MASK; + sh64_dcache_purge_coloured_phy_page(paddr, addr); + } while (pte++, addr += PAGE_SIZE, addr != end); + pte_unmap_unlock(pte - 1, ptl); +} +/****************************************************************************/ + +static void sh64_dcache_purge_user_range(struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + /* There are at least 5 choices for the implementation of this, with + pros (+), cons(-), comments(*): + + 1. ocbp each line in the range through the original user's ASID + + no lines spuriously evicted + - tlbmiss handling (must either handle faults on demand => extra + special-case code in tlbmiss critical path), or map the page in + advance (=> flush_tlb_range in advance to avoid multiple hits) + - ASID switching + - expensive for large ranges + + 2. temporarily map each page in the range to a special effective + address and ocbp through the temporary mapping; relies on the + fact that SH-5 OCB* always do TLB lookup and match on ptags (they + never look at the etags) + + no spurious evictions + - expensive for large ranges + * surely cheaper than (1) + + 3. walk all the lines in the cache, check the tags, if a match + occurs create a page mapping to ocbp the line through + + no spurious evictions + - tag inspection overhead + - (especially for small ranges) + - potential cost of setting up/tearing down page mapping for + every line that matches the range + * cost partly independent of range size + + 4. walk all the lines in the cache, check the tags, if a match + occurs use 4 * alloco to purge the line (+3 other probably + innocent victims) by natural eviction + + no tlb mapping overheads + - spurious evictions + - tag inspection overhead + + 5. implement like flush_cache_all + + no tag inspection overhead + - spurious evictions + - bad for small ranges + + (1) can be ruled out as more expensive than (2). (2) appears best + for small ranges. The choice between (3), (4) and (5) for large + ranges and the range size for the large/small boundary need + benchmarking to determine. + + For now use approach (2) for small ranges and (5) for large ones. + + */ + + int n_pages; + + n_pages = ((end - start) >> PAGE_SHIFT); + if (n_pages >= 64 || ((start ^ (end - 1)) & PMD_MASK)) { +#if 1 + sh64_dcache_purge_all(); +#else + unsigned long long set, way; + unsigned long mm_asid = mm->context & MMU_CONTEXT_ASID_MASK; + for (set = 0; set < cpu_data->dcache.sets; set++) { + unsigned long long set_base_config_addr = CACHE_OC_ADDRESS_ARRAY + (set << cpu_data->dcache.set_shift); + for (way = 0; way < cpu_data->dcache.ways; way++) { + unsigned long long config_addr = set_base_config_addr + (way << cpu_data->dcache.way_step_shift); + unsigned long long tag0; + unsigned long line_valid; + + asm __volatile__("getcfg %1, 0, %0" : "=r" (tag0) : "r" (config_addr)); + line_valid = tag0 & SH_CACHE_VALID; + if (line_valid) { + unsigned long cache_asid; + unsigned long epn; + + cache_asid = (tag0 & cpu_data->dcache.asid_mask) >> cpu_data->dcache.asid_shift; + /* The next line needs some + explanation. The virtual tags + encode bits [31:13] of the virtual + address, bit [12] of the 'tag' being + implied by the cache set index. */ + epn = (tag0 & cpu_data->dcache.epn_mask) | ((set & 0x80) << cpu_data->dcache.entry_shift); + + if ((cache_asid == mm_asid) && (start <= epn) && (epn < end)) { + /* TODO : could optimise this + call by batching multiple + adjacent sets together. */ + sh64_dcache_purge_sets(set, 1); + break; /* Don't waste time inspecting other ways for this set */ + } + } + } + } +#endif + } else { + /* Small range, covered by a single page table page */ + start &= PAGE_MASK; /* should already be so */ + end = PAGE_ALIGN(end); /* should already be so */ + sh64_dcache_purge_user_pages(mm, start, end); + } + return; +} + +static void sh64_dcache_wback_current_user_range(unsigned long start, unsigned long end) +{ + unsigned long long aligned_start; + unsigned long long ull_end; + unsigned long long addr; + + ull_end = end; + + /* Just wback over the range using the natural addresses. TLB miss + handling will be OK (TBC) : the range has just been written to by + the signal frame setup code, so the PTEs must exist. + + Note, if we have CONFIG_PREEMPT and get preempted inside this loop, + it doesn't matter, even if the pid->ASID mapping changes whilst + we're away. In that case the cache will have been flushed when the + mapping was renewed. So the writebacks below will be nugatory (and + we'll doubtless have to fault the TLB entry/ies in again with the + new ASID), but it's a rare case. + */ + aligned_start = start & L1_CACHE_ALIGN_MASK; + addr = aligned_start; + while (addr < ull_end) { + asm __volatile__ ("ocbwb %0, 0" : : "r" (addr)); + addr += L1_CACHE_BYTES; + } +} + +/****************************************************************************/ + +/* These *MUST* lie in an area of virtual address space that's otherwise unused. */ +#define UNIQUE_EADDR_START 0xe0000000UL +#define UNIQUE_EADDR_END 0xe8000000UL + +static unsigned long sh64_make_unique_eaddr(unsigned long user_eaddr, unsigned long paddr) +{ + /* Given a physical address paddr, and a user virtual address + user_eaddr which will eventually be mapped to it, create a one-off + kernel-private eaddr mapped to the same paddr. This is used for + creating special destination pages for copy_user_page and + clear_user_page */ + + static unsigned long current_pointer = UNIQUE_EADDR_START; + unsigned long coloured_pointer; + + if (current_pointer == UNIQUE_EADDR_END) { + sh64_dcache_purge_all(); + current_pointer = UNIQUE_EADDR_START; + } + + coloured_pointer = (current_pointer & ~CACHE_OC_SYN_MASK) | (user_eaddr & CACHE_OC_SYN_MASK); + sh64_setup_dtlb_cache_slot(coloured_pointer, get_asid(), paddr); + + current_pointer += (PAGE_SIZE << CACHE_OC_N_SYNBITS); + + return coloured_pointer; +} + +/****************************************************************************/ + +static void sh64_copy_user_page_coloured(void *to, void *from, unsigned long address) +{ + void *coloured_to; + + /* Discard any existing cache entries of the wrong colour. These are + present quite often, if the kernel has recently used the page + internally, then given it up, then it's been allocated to the user. + */ + sh64_dcache_purge_coloured_phy_page(__pa(to), (unsigned long) to); + + coloured_to = (void *) sh64_make_unique_eaddr(address, __pa(to)); + sh64_page_copy(from, coloured_to); + + sh64_teardown_dtlb_cache_slot(); +} + +static void sh64_clear_user_page_coloured(void *to, unsigned long address) +{ + void *coloured_to; + + /* Discard any existing kernel-originated lines of the wrong colour (as + above) */ + sh64_dcache_purge_coloured_phy_page(__pa(to), (unsigned long) to); + + coloured_to = (void *) sh64_make_unique_eaddr(address, __pa(to)); + sh64_page_clear(coloured_to); + + sh64_teardown_dtlb_cache_slot(); +} + +#endif /* !CONFIG_DCACHE_DISABLED */ + +/****************************************************************************/ + +/*########################################################################## + EXTERNALLY CALLABLE API. + ##########################################################################*/ + +/* These functions are described in Documentation/cachetlb.txt. + Each one of these functions varies in behaviour depending on whether the + I-cache and/or D-cache are configured out. + + Note that the Linux term 'flush' corresponds to what is termed 'purge' in + the sh/sh64 jargon for the D-cache, i.e. write back dirty data then + invalidate the cache lines, and 'invalidate' for the I-cache. + */ + +#undef FLUSH_TRACE + +void flush_cache_all(void) +{ + /* Invalidate the entire contents of both caches, after writing back to + memory any dirty data from the D-cache. */ + sh64_dcache_purge_all(); + sh64_icache_inv_all(); +} + +/****************************************************************************/ + +void flush_cache_mm(struct mm_struct *mm) +{ + /* Invalidate an entire user-address space from both caches, after + writing back dirty data (e.g. for shared mmap etc). */ + + /* This could be coded selectively by inspecting all the tags then + doing 4*alloco on any set containing a match (as for + flush_cache_range), but fork/exit/execve (where this is called from) + are expensive anyway. */ + + /* Have to do a purge here, despite the comments re I-cache below. + There could be odd-coloured dirty data associated with the mm still + in the cache - if this gets written out through natural eviction + after the kernel has reused the page there will be chaos. + */ + + sh64_dcache_purge_all(); + + /* The mm being torn down won't ever be active again, so any Icache + lines tagged with its ASID won't be visible for the rest of the + lifetime of this ASID cycle. Before the ASID gets reused, there + will be a flush_cache_all. Hence we don't need to touch the + I-cache. This is similar to the lack of action needed in + flush_tlb_mm - see fault.c. */ +} + +/****************************************************************************/ + +void flush_cache_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) +{ + struct mm_struct *mm = vma->vm_mm; + + /* Invalidate (from both caches) the range [start,end) of virtual + addresses from the user address space specified by mm, after writing + back any dirty data. + + Note, 'end' is 1 byte beyond the end of the range to flush. */ + + sh64_dcache_purge_user_range(mm, start, end); + sh64_icache_inv_user_page_range(mm, start, end); +} + +/****************************************************************************/ + +void flush_cache_page(struct vm_area_struct *vma, unsigned long eaddr, unsigned long pfn) +{ + /* Invalidate any entries in either cache for the vma within the user + address space vma->vm_mm for the page starting at virtual address + 'eaddr'. This seems to be used primarily in breaking COW. Note, + the I-cache must be searched too in case the page in question is + both writable and being executed from (e.g. stack trampolines.) + + Note, this is called with pte lock held. + */ + + sh64_dcache_purge_phy_page(pfn << PAGE_SHIFT); + + if (vma->vm_flags & VM_EXEC) { + sh64_icache_inv_user_page(vma, eaddr); + } +} + +/****************************************************************************/ + +#ifndef CONFIG_DCACHE_DISABLED + +void copy_user_page(void *to, void *from, unsigned long address, struct page *page) +{ + /* 'from' and 'to' are kernel virtual addresses (within the superpage + mapping of the physical RAM). 'address' is the user virtual address + where the copy 'to' will be mapped after. This allows a custom + mapping to be used to ensure that the new copy is placed in the + right cache sets for the user to see it without having to bounce it + out via memory. Note however : the call to flush_page_to_ram in + (generic)/mm/memory.c:(break_cow) undoes all this good work in that one + very important case! + + TBD : can we guarantee that on every call, any cache entries for + 'from' are in the same colour sets as 'address' also? i.e. is this + always used just to deal with COW? (I suspect not). */ + + /* There are two possibilities here for when the page 'from' was last accessed: + * by the kernel : this is OK, no purge required. + * by the/a user (e.g. for break_COW) : need to purge. + + If the potential user mapping at 'address' is the same colour as + 'from' there is no need to purge any cache lines from the 'from' + page mapped into cache sets of colour 'address'. (The copy will be + accessing the page through 'from'). + */ + + if (((address ^ (unsigned long) from) & CACHE_OC_SYN_MASK) != 0) { + sh64_dcache_purge_coloured_phy_page(__pa(from), address); + } + + if (((address ^ (unsigned long) to) & CACHE_OC_SYN_MASK) == 0) { + /* No synonym problem on destination */ + sh64_page_copy(from, to); + } else { + sh64_copy_user_page_coloured(to, from, address); + } + + /* Note, don't need to flush 'from' page from the cache again - it's + done anyway by the generic code */ +} + +void clear_user_page(void *to, unsigned long address, struct page *page) +{ + /* 'to' is a kernel virtual address (within the superpage + mapping of the physical RAM). 'address' is the user virtual address + where the 'to' page will be mapped after. This allows a custom + mapping to be used to ensure that the new copy is placed in the + right cache sets for the user to see it without having to bounce it + out via memory. + */ + + if (((address ^ (unsigned long) to) & CACHE_OC_SYN_MASK) == 0) { + /* No synonym problem on destination */ + sh64_page_clear(to); + } else { + sh64_clear_user_page_coloured(to, address); + } +} + +#endif /* !CONFIG_DCACHE_DISABLED */ + +/****************************************************************************/ + +void flush_dcache_page(struct page *page) +{ + sh64_dcache_purge_phy_page(page_to_phys(page)); + wmb(); +} + +/****************************************************************************/ + +void flush_icache_range(unsigned long start, unsigned long end) +{ + /* Flush the range [start,end] of kernel virtual adddress space from + the I-cache. The corresponding range must be purged from the + D-cache also because the SH-5 doesn't have cache snooping between + the caches. The addresses will be visible through the superpage + mapping, therefore it's guaranteed that there no cache entries for + the range in cache sets of the wrong colour. + + Primarily used for cohering the I-cache after a module has + been loaded. */ + + /* We also make sure to purge the same range from the D-cache since + flush_page_to_ram() won't be doing this for us! */ + + sh64_dcache_purge_kernel_range(start, end); + wmb(); + sh64_icache_inv_kernel_range(start, end); +} + +/****************************************************************************/ + +void flush_icache_user_range(struct vm_area_struct *vma, + struct page *page, unsigned long addr, int len) +{ + /* Flush the range of user (defined by vma->vm_mm) address space + starting at 'addr' for 'len' bytes from the cache. The range does + not straddle a page boundary, the unique physical page containing + the range is 'page'. This seems to be used mainly for invalidating + an address range following a poke into the program text through the + ptrace() call from another process (e.g. for BRK instruction + insertion). */ + + sh64_dcache_purge_coloured_phy_page(page_to_phys(page), addr); + mb(); + + if (vma->vm_flags & VM_EXEC) { + sh64_icache_inv_user_small_range(vma->vm_mm, addr, len); + } +} + +/*########################################################################## + ARCH/SH64 PRIVATE CALLABLE API. + ##########################################################################*/ + +void flush_cache_sigtramp(unsigned long start, unsigned long end) +{ + /* For the address range [start,end), write back the data from the + D-cache and invalidate the corresponding region of the I-cache for + the current process. Used to flush signal trampolines on the stack + to make them executable. */ + + sh64_dcache_wback_current_user_range(start, end); + wmb(); + sh64_icache_inv_current_user_range(start, end); +} + diff --git a/arch/sh/mm/tlb-sh5.c b/arch/sh/mm/tlb-sh5.c new file mode 100644 index 000000000000..d517e7d70340 --- /dev/null +++ b/arch/sh/mm/tlb-sh5.c @@ -0,0 +1,166 @@ +/* + * arch/sh64/mm/tlb.c + * + * Copyright (C) 2003 Paul Mundt + * Copyright (C) 2003 Richard Curnow + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + */ +#include +#include +#include +#include +#include + +/** + * sh64_tlb_init + * + * Perform initial setup for the DTLB and ITLB. + */ +int __init sh64_tlb_init(void) +{ + /* Assign some sane DTLB defaults */ + cpu_data->dtlb.entries = 64; + cpu_data->dtlb.step = 0x10; + + cpu_data->dtlb.first = DTLB_FIXED | cpu_data->dtlb.step; + cpu_data->dtlb.next = cpu_data->dtlb.first; + + cpu_data->dtlb.last = DTLB_FIXED | + ((cpu_data->dtlb.entries - 1) * + cpu_data->dtlb.step); + + /* And again for the ITLB */ + cpu_data->itlb.entries = 64; + cpu_data->itlb.step = 0x10; + + cpu_data->itlb.first = ITLB_FIXED | cpu_data->itlb.step; + cpu_data->itlb.next = cpu_data->itlb.first; + cpu_data->itlb.last = ITLB_FIXED | + ((cpu_data->itlb.entries - 1) * + cpu_data->itlb.step); + + return 0; +} + +/** + * sh64_next_free_dtlb_entry + * + * Find the next available DTLB entry + */ +unsigned long long sh64_next_free_dtlb_entry(void) +{ + return cpu_data->dtlb.next; +} + +/** + * sh64_get_wired_dtlb_entry + * + * Allocate a wired (locked-in) entry in the DTLB + */ +unsigned long long sh64_get_wired_dtlb_entry(void) +{ + unsigned long long entry = sh64_next_free_dtlb_entry(); + + cpu_data->dtlb.first += cpu_data->dtlb.step; + cpu_data->dtlb.next += cpu_data->dtlb.step; + + return entry; +} + +/** + * sh64_put_wired_dtlb_entry + * + * @entry: Address of TLB slot. + * + * Free a wired (locked-in) entry in the DTLB. + * + * Works like a stack, last one to allocate must be first one to free. + */ +int sh64_put_wired_dtlb_entry(unsigned long long entry) +{ + __flush_tlb_slot(entry); + + /* + * We don't do any particularly useful tracking of wired entries, + * so this approach works like a stack .. last one to be allocated + * has to be the first one to be freed. + * + * We could potentially load wired entries into a list and work on + * rebalancing the list periodically (which also entails moving the + * contents of a TLB entry) .. though I have a feeling that this is + * more trouble than it's worth. + */ + + /* + * Entry must be valid .. we don't want any ITLB addresses! + */ + if (entry <= DTLB_FIXED) + return -EINVAL; + + /* + * Next, check if we're within range to be freed. (ie, must be the + * entry beneath the first 'free' entry! + */ + if (entry < (cpu_data->dtlb.first - cpu_data->dtlb.step)) + return -EINVAL; + + /* If we are, then bring this entry back into the list */ + cpu_data->dtlb.first -= cpu_data->dtlb.step; + cpu_data->dtlb.next = entry; + + return 0; +} + +/** + * sh64_setup_tlb_slot + * + * @config_addr: Address of TLB slot. + * @eaddr: Virtual address. + * @asid: Address Space Identifier. + * @paddr: Physical address. + * + * Load up a virtual<->physical translation for @eaddr<->@paddr in the + * pre-allocated TLB slot @config_addr (see sh64_get_wired_dtlb_entry). + */ +inline void sh64_setup_tlb_slot(unsigned long long config_addr, + unsigned long eaddr, + unsigned long asid, + unsigned long paddr) +{ + unsigned long long pteh, ptel; + + /* Sign extension */ +#if (NEFF == 32) + pteh = (unsigned long long)(signed long long)(signed long) eaddr; +#else +#error "Can't sign extend more than 32 bits yet" +#endif + pteh &= PAGE_MASK; + pteh |= (asid << PTEH_ASID_SHIFT) | PTEH_VALID; +#if (NEFF == 32) + ptel = (unsigned long long)(signed long long)(signed long) paddr; +#else +#error "Can't sign extend more than 32 bits yet" +#endif + ptel &= PAGE_MASK; + ptel |= (_PAGE_CACHABLE | _PAGE_READ | _PAGE_WRITE); + + asm volatile("putcfg %0, 1, %1\n\t" + "putcfg %0, 0, %2\n" + : : "r" (config_addr), "r" (ptel), "r" (pteh)); +} + +/** + * sh64_teardown_tlb_slot + * + * @config_addr: Address of TLB slot. + * + * Teardown any existing mapping in the TLB slot @config_addr. + */ +inline void sh64_teardown_tlb_slot(unsigned long long config_addr) + __attribute__ ((alias("__flush_tlb_slot"))); + -- cgit v1.2.3 From 1e1ed39faec635b109ff8c516377310600623674 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Wed, 21 Nov 2007 17:58:13 +0900 Subject: sh: Copy over special CFLAGS for SH-5 TLB miss. Signed-off-by: Paul Mundt --- arch/sh/mm/Makefile_64 | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/Makefile_64 b/arch/sh/mm/Makefile_64 index 5ff83583593e..2731cf1c4e0a 100644 --- a/arch/sh/mm/Makefile_64 +++ b/arch/sh/mm/Makefile_64 @@ -17,3 +17,28 @@ obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_NUMA) += numa.o EXTRA_CFLAGS += -Werror + +# Special flags for fault_64.o. This puts restrictions on the number of +# caller-save registers that the compiler can target when building this file. +# This is required because the code is called from a context in entry.S where +# very few registers have been saved in the exception handler (for speed +# reasons). +# The caller save registers that have been saved and which can be used are +# r2,r3,r4,r5 : argument passing +# r15, r18 : SP and LINK +# tr0-4 : allow all caller-save TR's. The compiler seems to be able to make +# use of them, so it's probably beneficial to performance to save them +# and have them available for it. +# +# The resources not listed below are callee save, i.e. the compiler is free to +# use any of them and will spill them to the stack itself. + +CFLAGS_fault_64.o += -ffixed-r7 \ + -ffixed-r8 -ffixed-r9 -ffixed-r10 -ffixed-r11 -ffixed-r12 \ + -ffixed-r13 -ffixed-r14 -ffixed-r16 -ffixed-r17 -ffixed-r19 \ + -ffixed-r20 -ffixed-r21 -ffixed-r22 -ffixed-r23 \ + -ffixed-r24 -ffixed-r25 -ffixed-r26 -ffixed-r27 \ + -ffixed-r36 -ffixed-r37 -ffixed-r38 -ffixed-r39 -ffixed-r40 \ + -ffixed-r41 -ffixed-r42 -ffixed-r43 \ + -ffixed-r60 -ffixed-r61 -ffixed-r62 \ + -fomit-frame-pointer -- cgit v1.2.3 From efddb4c882a64b7ca44b2a089d20020b30a2c6dd Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Wed, 21 Nov 2007 22:30:58 +0900 Subject: sh: Rename the _32 and _64 TLB flush variants. Signed-off-by: Paul Mundt --- arch/sh/mm/Makefile_32 | 2 +- arch/sh/mm/Makefile_64 | 2 +- arch/sh/mm/tlb-flush_32.c | 140 -------------- arch/sh/mm/tlb-flush_64.c | 475 ---------------------------------------------- arch/sh/mm/tlbflush_32.c | 140 ++++++++++++++ arch/sh/mm/tlbflush_64.c | 475 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 617 insertions(+), 617 deletions(-) delete mode 100644 arch/sh/mm/tlb-flush_32.c delete mode 100644 arch/sh/mm/tlb-flush_64.c create mode 100644 arch/sh/mm/tlbflush_32.c create mode 100644 arch/sh/mm/tlbflush_64.c (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/Makefile_32 b/arch/sh/mm/Makefile_32 index fc089c0cdbe9..095abd14592f 100644 --- a/arch/sh/mm/Makefile_32 +++ b/arch/sh/mm/Makefile_32 @@ -12,7 +12,7 @@ obj-$(CONFIG_SH7705_CACHE_32KB) += cache-sh7705.o endif mmu-y := tlb-nommu.o pg-nommu.o -mmu-$(CONFIG_MMU) := fault_32.o clear_page.o copy_page.o tlb-flush_32.o \ +mmu-$(CONFIG_MMU) := fault_32.o clear_page.o copy_page.o tlbflush_32.o \ ioremap_32.o obj-y += $(mmu-y) diff --git a/arch/sh/mm/Makefile_64 b/arch/sh/mm/Makefile_64 index 2731cf1c4e0a..cbd6aa33c5ac 100644 --- a/arch/sh/mm/Makefile_64 +++ b/arch/sh/mm/Makefile_64 @@ -5,7 +5,7 @@ obj-y := init.o extable_64.o consistent.o mmu-y := tlb-nommu.o pg-nommu.o -mmu-$(CONFIG_MMU) := fault_64.o ioremap_64.o tlb-flush_64.o tlb-sh5.o +mmu-$(CONFIG_MMU) := fault_64.o ioremap_64.o tlbflush_64.o tlb-sh5.o ifndef CONFIG_CACHE_OFF obj-y += cache-sh5.o diff --git a/arch/sh/mm/tlb-flush_32.c b/arch/sh/mm/tlb-flush_32.c deleted file mode 100644 index 6f45c1f8a7fe..000000000000 --- a/arch/sh/mm/tlb-flush_32.c +++ /dev/null @@ -1,140 +0,0 @@ -/* - * TLB flushing operations for SH with an MMU. - * - * Copyright (C) 1999 Niibe Yutaka - * Copyright (C) 2003 Paul Mundt - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - */ -#include -#include -#include - -void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) -{ - unsigned int cpu = smp_processor_id(); - - if (vma->vm_mm && cpu_context(cpu, vma->vm_mm) != NO_CONTEXT) { - unsigned long flags; - unsigned long asid; - unsigned long saved_asid = MMU_NO_ASID; - - asid = cpu_asid(cpu, vma->vm_mm); - page &= PAGE_MASK; - - local_irq_save(flags); - if (vma->vm_mm != current->mm) { - saved_asid = get_asid(); - set_asid(asid); - } - local_flush_tlb_one(asid, page); - if (saved_asid != MMU_NO_ASID) - set_asid(saved_asid); - local_irq_restore(flags); - } -} - -void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end) -{ - struct mm_struct *mm = vma->vm_mm; - unsigned int cpu = smp_processor_id(); - - if (cpu_context(cpu, mm) != NO_CONTEXT) { - unsigned long flags; - int size; - - local_irq_save(flags); - size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; - if (size > (MMU_NTLB_ENTRIES/4)) { /* Too many TLB to flush */ - cpu_context(cpu, mm) = NO_CONTEXT; - if (mm == current->mm) - activate_context(mm, cpu); - } else { - unsigned long asid; - unsigned long saved_asid = MMU_NO_ASID; - - asid = cpu_asid(cpu, mm); - start &= PAGE_MASK; - end += (PAGE_SIZE - 1); - end &= PAGE_MASK; - if (mm != current->mm) { - saved_asid = get_asid(); - set_asid(asid); - } - while (start < end) { - local_flush_tlb_one(asid, start); - start += PAGE_SIZE; - } - if (saved_asid != MMU_NO_ASID) - set_asid(saved_asid); - } - local_irq_restore(flags); - } -} - -void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) -{ - unsigned int cpu = smp_processor_id(); - unsigned long flags; - int size; - - local_irq_save(flags); - size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; - if (size > (MMU_NTLB_ENTRIES/4)) { /* Too many TLB to flush */ - local_flush_tlb_all(); - } else { - unsigned long asid; - unsigned long saved_asid = get_asid(); - - asid = cpu_asid(cpu, &init_mm); - start &= PAGE_MASK; - end += (PAGE_SIZE - 1); - end &= PAGE_MASK; - set_asid(asid); - while (start < end) { - local_flush_tlb_one(asid, start); - start += PAGE_SIZE; - } - set_asid(saved_asid); - } - local_irq_restore(flags); -} - -void local_flush_tlb_mm(struct mm_struct *mm) -{ - unsigned int cpu = smp_processor_id(); - - /* Invalidate all TLB of this process. */ - /* Instead of invalidating each TLB, we get new MMU context. */ - if (cpu_context(cpu, mm) != NO_CONTEXT) { - unsigned long flags; - - local_irq_save(flags); - cpu_context(cpu, mm) = NO_CONTEXT; - if (mm == current->mm) - activate_context(mm, cpu); - local_irq_restore(flags); - } -} - -void local_flush_tlb_all(void) -{ - unsigned long flags, status; - - /* - * Flush all the TLB. - * - * Write to the MMU control register's bit: - * TF-bit for SH-3, TI-bit for SH-4. - * It's same position, bit #2. - */ - local_irq_save(flags); - status = ctrl_inl(MMUCR); - status |= 0x04; - ctrl_outl(status, MMUCR); - ctrl_barrier(); - local_irq_restore(flags); -} diff --git a/arch/sh/mm/tlb-flush_64.c b/arch/sh/mm/tlb-flush_64.c deleted file mode 100644 index 2a98c9ec88ff..000000000000 --- a/arch/sh/mm/tlb-flush_64.c +++ /dev/null @@ -1,475 +0,0 @@ -/* - * arch/sh/mm/tlb-flush_64.c - * - * Copyright (C) 2000, 2001 Paolo Alberelli - * Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes) - * Copyright (C) 2003 Paul Mundt - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -extern void die(const char *,struct pt_regs *,long); - -#define PFLAG(val,flag) (( (val) & (flag) ) ? #flag : "" ) -#define PPROT(flag) PFLAG(pgprot_val(prot),flag) - -static inline void print_prots(pgprot_t prot) -{ - printk("prot is 0x%08lx\n",pgprot_val(prot)); - - printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ), - PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER)); -} - -static inline void print_vma(struct vm_area_struct *vma) -{ - printk("vma start 0x%08lx\n", vma->vm_start); - printk("vma end 0x%08lx\n", vma->vm_end); - - print_prots(vma->vm_page_prot); - printk("vm_flags 0x%08lx\n", vma->vm_flags); -} - -static inline void print_task(struct task_struct *tsk) -{ - printk("Task pid %d\n", task_pid_nr(tsk)); -} - -static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address) -{ - pgd_t *dir; - pud_t *pud; - pmd_t *pmd; - pte_t *pte; - pte_t entry; - - dir = pgd_offset(mm, address); - if (pgd_none(*dir)) - return NULL; - - pud = pud_offset(dir, address); - if (pud_none(*pud)) - return NULL; - - pmd = pmd_offset(pud, address); - if (pmd_none(*pmd)) - return NULL; - - pte = pte_offset_kernel(pmd, address); - entry = *pte; - if (pte_none(entry) || !pte_present(entry)) - return NULL; - - return pte; -} - -/* - * This routine handles page faults. It determines the address, - * and the problem, and then passes it off to one of the appropriate - * routines. - */ -asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess, - unsigned long textaccess, unsigned long address) -{ - struct task_struct *tsk; - struct mm_struct *mm; - struct vm_area_struct * vma; - const struct exception_table_entry *fixup; - pte_t *pte; - int fault; - - /* SIM - * Note this is now called with interrupts still disabled - * This is to cope with being called for a missing IO port - * address with interrupts disabled. This should be fixed as - * soon as we have a better 'fast path' miss handler. - * - * Plus take care how you try and debug this stuff. - * For example, writing debug data to a port which you - * have just faulted on is not going to work. - */ - - tsk = current; - mm = tsk->mm; - - /* Not an IO address, so reenable interrupts */ - local_irq_enable(); - - /* - * If we're in an interrupt or have no user - * context, we must not take the fault.. - */ - if (in_atomic() || !mm) - goto no_context; - - /* TLB misses upon some cache flushes get done under cli() */ - down_read(&mm->mmap_sem); - - vma = find_vma(mm, address); - - if (!vma) { -#ifdef DEBUG_FAULT - print_task(tsk); - printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", - __FUNCTION__,__LINE__, - address,regs->pc,textaccess,writeaccess); - show_regs(regs); -#endif - goto bad_area; - } - if (vma->vm_start <= address) { - goto good_area; - } - - if (!(vma->vm_flags & VM_GROWSDOWN)) { -#ifdef DEBUG_FAULT - print_task(tsk); - printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", - __FUNCTION__,__LINE__, - address,regs->pc,textaccess,writeaccess); - show_regs(regs); - - print_vma(vma); -#endif - goto bad_area; - } - if (expand_stack(vma, address)) { -#ifdef DEBUG_FAULT - print_task(tsk); - printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", - __FUNCTION__,__LINE__, - address,regs->pc,textaccess,writeaccess); - show_regs(regs); -#endif - goto bad_area; - } -/* - * Ok, we have a good vm_area for this memory access, so - * we can handle it.. - */ -good_area: - if (textaccess) { - if (!(vma->vm_flags & VM_EXEC)) - goto bad_area; - } else { - if (writeaccess) { - if (!(vma->vm_flags & VM_WRITE)) - goto bad_area; - } else { - if (!(vma->vm_flags & VM_READ)) - goto bad_area; - } - } - - /* - * If for any reason at all we couldn't handle the fault, - * make sure we exit gracefully rather than endlessly redo - * the fault. - */ -survive: - fault = handle_mm_fault(mm, vma, address, writeaccess); - if (unlikely(fault & VM_FAULT_ERROR)) { - if (fault & VM_FAULT_OOM) - goto out_of_memory; - else if (fault & VM_FAULT_SIGBUS) - goto do_sigbus; - BUG(); - } - if (fault & VM_FAULT_MAJOR) - tsk->maj_flt++; - else - tsk->min_flt++; - - /* If we get here, the page fault has been handled. Do the TLB refill - now from the newly-setup PTE, to avoid having to fault again right - away on the same instruction. */ - pte = lookup_pte (mm, address); - if (!pte) { - /* From empirical evidence, we can get here, due to - !pte_present(pte). (e.g. if a swap-in occurs, and the page - is swapped back out again before the process that wanted it - gets rescheduled?) */ - goto no_pte; - } - - __do_tlb_refill(address, textaccess, pte); - -no_pte: - - up_read(&mm->mmap_sem); - return; - -/* - * Something tried to access memory that isn't in our memory map.. - * Fix it, but check if it's kernel or user first.. - */ -bad_area: -#ifdef DEBUG_FAULT - printk("fault:bad area\n"); -#endif - up_read(&mm->mmap_sem); - - if (user_mode(regs)) { - static int count=0; - siginfo_t info; - if (count < 4) { - /* This is really to help debug faults when starting - * usermode, so only need a few */ - count++; - printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n", - address, task_pid_nr(current), current->comm, - (unsigned long) regs->pc); -#if 0 - show_regs(regs); -#endif - } - if (is_global_init(tsk)) { - panic("INIT had user mode bad_area\n"); - } - tsk->thread.address = address; - tsk->thread.error_code = writeaccess; - info.si_signo = SIGSEGV; - info.si_errno = 0; - info.si_addr = (void *) address; - force_sig_info(SIGSEGV, &info, tsk); - return; - } - -no_context: -#ifdef DEBUG_FAULT - printk("fault:No context\n"); -#endif - /* Are we prepared to handle this kernel fault? */ - fixup = search_exception_tables(regs->pc); - if (fixup) { - regs->pc = fixup->fixup; - return; - } - -/* - * Oops. The kernel tried to access some bad page. We'll have to - * terminate things with extreme prejudice. - * - */ - if (address < PAGE_SIZE) - printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference"); - else - printk(KERN_ALERT "Unable to handle kernel paging request"); - printk(" at virtual address %08lx\n", address); - printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff); - die("Oops", regs, writeaccess); - do_exit(SIGKILL); - -/* - * We ran out of memory, or some other thing happened to us that made - * us unable to handle the page fault gracefully. - */ -out_of_memory: - if (is_global_init(current)) { - panic("INIT out of memory\n"); - yield(); - goto survive; - } - printk("fault:Out of memory\n"); - up_read(&mm->mmap_sem); - if (is_global_init(current)) { - yield(); - down_read(&mm->mmap_sem); - goto survive; - } - printk("VM: killing process %s\n", tsk->comm); - if (user_mode(regs)) - do_group_exit(SIGKILL); - goto no_context; - -do_sigbus: - printk("fault:Do sigbus\n"); - up_read(&mm->mmap_sem); - - /* - * Send a sigbus, regardless of whether we were in kernel - * or user mode. - */ - tsk->thread.address = address; - tsk->thread.error_code = writeaccess; - tsk->thread.trap_no = 14; - force_sig(SIGBUS, tsk); - - /* Kernel mode? Handle exceptions or die */ - if (!user_mode(regs)) - goto no_context; -} - -void update_mmu_cache(struct vm_area_struct * vma, - unsigned long address, pte_t pte) -{ - /* - * This appears to get called once for every pte entry that gets - * established => I don't think it's efficient to try refilling the - * TLBs with the pages - some may not get accessed even. Also, for - * executable pages, it is impossible to determine reliably here which - * TLB they should be mapped into (or both even). - * - * So, just do nothing here and handle faults on demand. In the - * TLBMISS handling case, the refill is now done anyway after the pte - * has been fixed up, so that deals with most useful cases. - */ -} - -void local_flush_tlb_one(unsigned long asid, unsigned long page) -{ - unsigned long long match, pteh=0, lpage; - unsigned long tlb; - - /* - * Sign-extend based on neff. - */ - lpage = (page & NEFF_SIGN) ? (page | NEFF_MASK) : page; - match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID; - match |= lpage; - - for_each_itlb_entry(tlb) { - asm volatile ("getcfg %1, 0, %0" - : "=r" (pteh) - : "r" (tlb) ); - - if (pteh == match) { - __flush_tlb_slot(tlb); - break; - } - } - - for_each_dtlb_entry(tlb) { - asm volatile ("getcfg %1, 0, %0" - : "=r" (pteh) - : "r" (tlb) ); - - if (pteh == match) { - __flush_tlb_slot(tlb); - break; - } - - } -} - -void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) -{ - unsigned long flags; - - if (vma->vm_mm) { - page &= PAGE_MASK; - local_irq_save(flags); - local_flush_tlb_one(get_asid(), page); - local_irq_restore(flags); - } -} - -void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end) -{ - unsigned long flags; - unsigned long long match, pteh=0, pteh_epn, pteh_low; - unsigned long tlb; - unsigned int cpu = smp_processor_id(); - struct mm_struct *mm; - - mm = vma->vm_mm; - if (cpu_context(cpu, mm) == NO_CONTEXT) - return; - - local_irq_save(flags); - - start &= PAGE_MASK; - end &= PAGE_MASK; - - match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID; - - /* Flush ITLB */ - for_each_itlb_entry(tlb) { - asm volatile ("getcfg %1, 0, %0" - : "=r" (pteh) - : "r" (tlb) ); - - pteh_epn = pteh & PAGE_MASK; - pteh_low = pteh & ~PAGE_MASK; - - if (pteh_low == match && pteh_epn >= start && pteh_epn <= end) - __flush_tlb_slot(tlb); - } - - /* Flush DTLB */ - for_each_dtlb_entry(tlb) { - asm volatile ("getcfg %1, 0, %0" - : "=r" (pteh) - : "r" (tlb) ); - - pteh_epn = pteh & PAGE_MASK; - pteh_low = pteh & ~PAGE_MASK; - - if (pteh_low == match && pteh_epn >= start && pteh_epn <= end) - __flush_tlb_slot(tlb); - } - - local_irq_restore(flags); -} - -void local_flush_tlb_mm(struct mm_struct *mm) -{ - unsigned long flags; - unsigned int cpu = smp_processor_id(); - - if (cpu_context(cpu, mm) == NO_CONTEXT) - return; - - local_irq_save(flags); - - cpu_context(cpu, mm) = NO_CONTEXT; - if (mm == current->mm) - activate_context(mm, cpu); - - local_irq_restore(flags); -} - -void local_flush_tlb_all(void) -{ - /* Invalidate all, including shared pages, excluding fixed TLBs */ - unsigned long flags, tlb; - - local_irq_save(flags); - - /* Flush each ITLB entry */ - for_each_itlb_entry(tlb) - __flush_tlb_slot(tlb); - - /* Flush each DTLB entry */ - for_each_dtlb_entry(tlb) - __flush_tlb_slot(tlb); - - local_irq_restore(flags); -} - -void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) -{ - /* FIXME: Optimize this later.. */ - flush_tlb_all(); -} diff --git a/arch/sh/mm/tlbflush_32.c b/arch/sh/mm/tlbflush_32.c new file mode 100644 index 000000000000..6f45c1f8a7fe --- /dev/null +++ b/arch/sh/mm/tlbflush_32.c @@ -0,0 +1,140 @@ +/* + * TLB flushing operations for SH with an MMU. + * + * Copyright (C) 1999 Niibe Yutaka + * Copyright (C) 2003 Paul Mundt + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include +#include +#include + +void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + unsigned int cpu = smp_processor_id(); + + if (vma->vm_mm && cpu_context(cpu, vma->vm_mm) != NO_CONTEXT) { + unsigned long flags; + unsigned long asid; + unsigned long saved_asid = MMU_NO_ASID; + + asid = cpu_asid(cpu, vma->vm_mm); + page &= PAGE_MASK; + + local_irq_save(flags); + if (vma->vm_mm != current->mm) { + saved_asid = get_asid(); + set_asid(asid); + } + local_flush_tlb_one(asid, page); + if (saved_asid != MMU_NO_ASID) + set_asid(saved_asid); + local_irq_restore(flags); + } +} + +void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned int cpu = smp_processor_id(); + + if (cpu_context(cpu, mm) != NO_CONTEXT) { + unsigned long flags; + int size; + + local_irq_save(flags); + size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; + if (size > (MMU_NTLB_ENTRIES/4)) { /* Too many TLB to flush */ + cpu_context(cpu, mm) = NO_CONTEXT; + if (mm == current->mm) + activate_context(mm, cpu); + } else { + unsigned long asid; + unsigned long saved_asid = MMU_NO_ASID; + + asid = cpu_asid(cpu, mm); + start &= PAGE_MASK; + end += (PAGE_SIZE - 1); + end &= PAGE_MASK; + if (mm != current->mm) { + saved_asid = get_asid(); + set_asid(asid); + } + while (start < end) { + local_flush_tlb_one(asid, start); + start += PAGE_SIZE; + } + if (saved_asid != MMU_NO_ASID) + set_asid(saved_asid); + } + local_irq_restore(flags); + } +} + +void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + unsigned int cpu = smp_processor_id(); + unsigned long flags; + int size; + + local_irq_save(flags); + size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; + if (size > (MMU_NTLB_ENTRIES/4)) { /* Too many TLB to flush */ + local_flush_tlb_all(); + } else { + unsigned long asid; + unsigned long saved_asid = get_asid(); + + asid = cpu_asid(cpu, &init_mm); + start &= PAGE_MASK; + end += (PAGE_SIZE - 1); + end &= PAGE_MASK; + set_asid(asid); + while (start < end) { + local_flush_tlb_one(asid, start); + start += PAGE_SIZE; + } + set_asid(saved_asid); + } + local_irq_restore(flags); +} + +void local_flush_tlb_mm(struct mm_struct *mm) +{ + unsigned int cpu = smp_processor_id(); + + /* Invalidate all TLB of this process. */ + /* Instead of invalidating each TLB, we get new MMU context. */ + if (cpu_context(cpu, mm) != NO_CONTEXT) { + unsigned long flags; + + local_irq_save(flags); + cpu_context(cpu, mm) = NO_CONTEXT; + if (mm == current->mm) + activate_context(mm, cpu); + local_irq_restore(flags); + } +} + +void local_flush_tlb_all(void) +{ + unsigned long flags, status; + + /* + * Flush all the TLB. + * + * Write to the MMU control register's bit: + * TF-bit for SH-3, TI-bit for SH-4. + * It's same position, bit #2. + */ + local_irq_save(flags); + status = ctrl_inl(MMUCR); + status |= 0x04; + ctrl_outl(status, MMUCR); + ctrl_barrier(); + local_irq_restore(flags); +} diff --git a/arch/sh/mm/tlbflush_64.c b/arch/sh/mm/tlbflush_64.c new file mode 100644 index 000000000000..2a98c9ec88ff --- /dev/null +++ b/arch/sh/mm/tlbflush_64.c @@ -0,0 +1,475 @@ +/* + * arch/sh/mm/tlb-flush_64.c + * + * Copyright (C) 2000, 2001 Paolo Alberelli + * Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes) + * Copyright (C) 2003 Paul Mundt + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern void die(const char *,struct pt_regs *,long); + +#define PFLAG(val,flag) (( (val) & (flag) ) ? #flag : "" ) +#define PPROT(flag) PFLAG(pgprot_val(prot),flag) + +static inline void print_prots(pgprot_t prot) +{ + printk("prot is 0x%08lx\n",pgprot_val(prot)); + + printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ), + PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER)); +} + +static inline void print_vma(struct vm_area_struct *vma) +{ + printk("vma start 0x%08lx\n", vma->vm_start); + printk("vma end 0x%08lx\n", vma->vm_end); + + print_prots(vma->vm_page_prot); + printk("vm_flags 0x%08lx\n", vma->vm_flags); +} + +static inline void print_task(struct task_struct *tsk) +{ + printk("Task pid %d\n", task_pid_nr(tsk)); +} + +static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address) +{ + pgd_t *dir; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + pte_t entry; + + dir = pgd_offset(mm, address); + if (pgd_none(*dir)) + return NULL; + + pud = pud_offset(dir, address); + if (pud_none(*pud)) + return NULL; + + pmd = pmd_offset(pud, address); + if (pmd_none(*pmd)) + return NULL; + + pte = pte_offset_kernel(pmd, address); + entry = *pte; + if (pte_none(entry) || !pte_present(entry)) + return NULL; + + return pte; +} + +/* + * This routine handles page faults. It determines the address, + * and the problem, and then passes it off to one of the appropriate + * routines. + */ +asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess, + unsigned long textaccess, unsigned long address) +{ + struct task_struct *tsk; + struct mm_struct *mm; + struct vm_area_struct * vma; + const struct exception_table_entry *fixup; + pte_t *pte; + int fault; + + /* SIM + * Note this is now called with interrupts still disabled + * This is to cope with being called for a missing IO port + * address with interrupts disabled. This should be fixed as + * soon as we have a better 'fast path' miss handler. + * + * Plus take care how you try and debug this stuff. + * For example, writing debug data to a port which you + * have just faulted on is not going to work. + */ + + tsk = current; + mm = tsk->mm; + + /* Not an IO address, so reenable interrupts */ + local_irq_enable(); + + /* + * If we're in an interrupt or have no user + * context, we must not take the fault.. + */ + if (in_atomic() || !mm) + goto no_context; + + /* TLB misses upon some cache flushes get done under cli() */ + down_read(&mm->mmap_sem); + + vma = find_vma(mm, address); + + if (!vma) { +#ifdef DEBUG_FAULT + print_task(tsk); + printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", + __FUNCTION__,__LINE__, + address,regs->pc,textaccess,writeaccess); + show_regs(regs); +#endif + goto bad_area; + } + if (vma->vm_start <= address) { + goto good_area; + } + + if (!(vma->vm_flags & VM_GROWSDOWN)) { +#ifdef DEBUG_FAULT + print_task(tsk); + printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", + __FUNCTION__,__LINE__, + address,regs->pc,textaccess,writeaccess); + show_regs(regs); + + print_vma(vma); +#endif + goto bad_area; + } + if (expand_stack(vma, address)) { +#ifdef DEBUG_FAULT + print_task(tsk); + printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", + __FUNCTION__,__LINE__, + address,regs->pc,textaccess,writeaccess); + show_regs(regs); +#endif + goto bad_area; + } +/* + * Ok, we have a good vm_area for this memory access, so + * we can handle it.. + */ +good_area: + if (textaccess) { + if (!(vma->vm_flags & VM_EXEC)) + goto bad_area; + } else { + if (writeaccess) { + if (!(vma->vm_flags & VM_WRITE)) + goto bad_area; + } else { + if (!(vma->vm_flags & VM_READ)) + goto bad_area; + } + } + + /* + * If for any reason at all we couldn't handle the fault, + * make sure we exit gracefully rather than endlessly redo + * the fault. + */ +survive: + fault = handle_mm_fault(mm, vma, address, writeaccess); + if (unlikely(fault & VM_FAULT_ERROR)) { + if (fault & VM_FAULT_OOM) + goto out_of_memory; + else if (fault & VM_FAULT_SIGBUS) + goto do_sigbus; + BUG(); + } + if (fault & VM_FAULT_MAJOR) + tsk->maj_flt++; + else + tsk->min_flt++; + + /* If we get here, the page fault has been handled. Do the TLB refill + now from the newly-setup PTE, to avoid having to fault again right + away on the same instruction. */ + pte = lookup_pte (mm, address); + if (!pte) { + /* From empirical evidence, we can get here, due to + !pte_present(pte). (e.g. if a swap-in occurs, and the page + is swapped back out again before the process that wanted it + gets rescheduled?) */ + goto no_pte; + } + + __do_tlb_refill(address, textaccess, pte); + +no_pte: + + up_read(&mm->mmap_sem); + return; + +/* + * Something tried to access memory that isn't in our memory map.. + * Fix it, but check if it's kernel or user first.. + */ +bad_area: +#ifdef DEBUG_FAULT + printk("fault:bad area\n"); +#endif + up_read(&mm->mmap_sem); + + if (user_mode(regs)) { + static int count=0; + siginfo_t info; + if (count < 4) { + /* This is really to help debug faults when starting + * usermode, so only need a few */ + count++; + printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n", + address, task_pid_nr(current), current->comm, + (unsigned long) regs->pc); +#if 0 + show_regs(regs); +#endif + } + if (is_global_init(tsk)) { + panic("INIT had user mode bad_area\n"); + } + tsk->thread.address = address; + tsk->thread.error_code = writeaccess; + info.si_signo = SIGSEGV; + info.si_errno = 0; + info.si_addr = (void *) address; + force_sig_info(SIGSEGV, &info, tsk); + return; + } + +no_context: +#ifdef DEBUG_FAULT + printk("fault:No context\n"); +#endif + /* Are we prepared to handle this kernel fault? */ + fixup = search_exception_tables(regs->pc); + if (fixup) { + regs->pc = fixup->fixup; + return; + } + +/* + * Oops. The kernel tried to access some bad page. We'll have to + * terminate things with extreme prejudice. + * + */ + if (address < PAGE_SIZE) + printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference"); + else + printk(KERN_ALERT "Unable to handle kernel paging request"); + printk(" at virtual address %08lx\n", address); + printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff); + die("Oops", regs, writeaccess); + do_exit(SIGKILL); + +/* + * We ran out of memory, or some other thing happened to us that made + * us unable to handle the page fault gracefully. + */ +out_of_memory: + if (is_global_init(current)) { + panic("INIT out of memory\n"); + yield(); + goto survive; + } + printk("fault:Out of memory\n"); + up_read(&mm->mmap_sem); + if (is_global_init(current)) { + yield(); + down_read(&mm->mmap_sem); + goto survive; + } + printk("VM: killing process %s\n", tsk->comm); + if (user_mode(regs)) + do_group_exit(SIGKILL); + goto no_context; + +do_sigbus: + printk("fault:Do sigbus\n"); + up_read(&mm->mmap_sem); + + /* + * Send a sigbus, regardless of whether we were in kernel + * or user mode. + */ + tsk->thread.address = address; + tsk->thread.error_code = writeaccess; + tsk->thread.trap_no = 14; + force_sig(SIGBUS, tsk); + + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) + goto no_context; +} + +void update_mmu_cache(struct vm_area_struct * vma, + unsigned long address, pte_t pte) +{ + /* + * This appears to get called once for every pte entry that gets + * established => I don't think it's efficient to try refilling the + * TLBs with the pages - some may not get accessed even. Also, for + * executable pages, it is impossible to determine reliably here which + * TLB they should be mapped into (or both even). + * + * So, just do nothing here and handle faults on demand. In the + * TLBMISS handling case, the refill is now done anyway after the pte + * has been fixed up, so that deals with most useful cases. + */ +} + +void local_flush_tlb_one(unsigned long asid, unsigned long page) +{ + unsigned long long match, pteh=0, lpage; + unsigned long tlb; + + /* + * Sign-extend based on neff. + */ + lpage = (page & NEFF_SIGN) ? (page | NEFF_MASK) : page; + match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID; + match |= lpage; + + for_each_itlb_entry(tlb) { + asm volatile ("getcfg %1, 0, %0" + : "=r" (pteh) + : "r" (tlb) ); + + if (pteh == match) { + __flush_tlb_slot(tlb); + break; + } + } + + for_each_dtlb_entry(tlb) { + asm volatile ("getcfg %1, 0, %0" + : "=r" (pteh) + : "r" (tlb) ); + + if (pteh == match) { + __flush_tlb_slot(tlb); + break; + } + + } +} + +void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + unsigned long flags; + + if (vma->vm_mm) { + page &= PAGE_MASK; + local_irq_save(flags); + local_flush_tlb_one(get_asid(), page); + local_irq_restore(flags); + } +} + +void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) +{ + unsigned long flags; + unsigned long long match, pteh=0, pteh_epn, pteh_low; + unsigned long tlb; + unsigned int cpu = smp_processor_id(); + struct mm_struct *mm; + + mm = vma->vm_mm; + if (cpu_context(cpu, mm) == NO_CONTEXT) + return; + + local_irq_save(flags); + + start &= PAGE_MASK; + end &= PAGE_MASK; + + match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID; + + /* Flush ITLB */ + for_each_itlb_entry(tlb) { + asm volatile ("getcfg %1, 0, %0" + : "=r" (pteh) + : "r" (tlb) ); + + pteh_epn = pteh & PAGE_MASK; + pteh_low = pteh & ~PAGE_MASK; + + if (pteh_low == match && pteh_epn >= start && pteh_epn <= end) + __flush_tlb_slot(tlb); + } + + /* Flush DTLB */ + for_each_dtlb_entry(tlb) { + asm volatile ("getcfg %1, 0, %0" + : "=r" (pteh) + : "r" (tlb) ); + + pteh_epn = pteh & PAGE_MASK; + pteh_low = pteh & ~PAGE_MASK; + + if (pteh_low == match && pteh_epn >= start && pteh_epn <= end) + __flush_tlb_slot(tlb); + } + + local_irq_restore(flags); +} + +void local_flush_tlb_mm(struct mm_struct *mm) +{ + unsigned long flags; + unsigned int cpu = smp_processor_id(); + + if (cpu_context(cpu, mm) == NO_CONTEXT) + return; + + local_irq_save(flags); + + cpu_context(cpu, mm) = NO_CONTEXT; + if (mm == current->mm) + activate_context(mm, cpu); + + local_irq_restore(flags); +} + +void local_flush_tlb_all(void) +{ + /* Invalidate all, including shared pages, excluding fixed TLBs */ + unsigned long flags, tlb; + + local_irq_save(flags); + + /* Flush each ITLB entry */ + for_each_itlb_entry(tlb) + __flush_tlb_slot(tlb); + + /* Flush each DTLB entry */ + for_each_dtlb_entry(tlb) + __flush_tlb_slot(tlb); + + local_irq_restore(flags); +} + +void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + /* FIXME: Optimize this later.. */ + flush_tlb_all(); +} -- cgit v1.2.3 From 9895f9429cb489ba271c06767531083ae4c4bcbe Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Wed, 21 Nov 2007 22:46:14 +0900 Subject: sh: clear/copy_page renames in lib and lib64. Signed-off-by: Paul Mundt --- arch/sh/mm/Makefile_32 | 3 +- arch/sh/mm/clear_page.S | 152 ------------------- arch/sh/mm/copy_page.S | 388 ------------------------------------------------ 3 files changed, 1 insertion(+), 542 deletions(-) delete mode 100644 arch/sh/mm/clear_page.S delete mode 100644 arch/sh/mm/copy_page.S (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/Makefile_32 b/arch/sh/mm/Makefile_32 index 095abd14592f..e295db60b91b 100644 --- a/arch/sh/mm/Makefile_32 +++ b/arch/sh/mm/Makefile_32 @@ -12,8 +12,7 @@ obj-$(CONFIG_SH7705_CACHE_32KB) += cache-sh7705.o endif mmu-y := tlb-nommu.o pg-nommu.o -mmu-$(CONFIG_MMU) := fault_32.o clear_page.o copy_page.o tlbflush_32.o \ - ioremap_32.o +mmu-$(CONFIG_MMU) := fault_32.o tlbflush_32.o ioremap_32.o obj-y += $(mmu-y) diff --git a/arch/sh/mm/clear_page.S b/arch/sh/mm/clear_page.S deleted file mode 100644 index 7a7c81ee3f01..000000000000 --- a/arch/sh/mm/clear_page.S +++ /dev/null @@ -1,152 +0,0 @@ -/* - * __clear_user_page, __clear_user, clear_page implementation of SuperH - * - * Copyright (C) 2001 Kaz Kojima - * Copyright (C) 2001, 2002 Niibe Yutaka - * Copyright (C) 2006 Paul Mundt - */ -#include -#include - -/* - * clear_page_slow - * @to: P1 address - * - * void clear_page_slow(void *to) - */ - -/* - * r0 --- scratch - * r4 --- to - * r5 --- to + PAGE_SIZE - */ -ENTRY(clear_page_slow) - mov r4,r5 - mov.l .Llimit,r0 - add r0,r5 - mov #0,r0 - ! -1: -#if defined(CONFIG_CPU_SH3) - mov.l r0,@r4 -#elif defined(CONFIG_CPU_SH4) - movca.l r0,@r4 - mov r4,r1 -#endif - add #32,r4 - mov.l r0,@-r4 - mov.l r0,@-r4 - mov.l r0,@-r4 - mov.l r0,@-r4 - mov.l r0,@-r4 - mov.l r0,@-r4 - mov.l r0,@-r4 -#if defined(CONFIG_CPU_SH4) - ocbwb @r1 -#endif - cmp/eq r5,r4 - bf/s 1b - add #28,r4 - ! - rts - nop -.Llimit: .long (PAGE_SIZE-28) - -ENTRY(__clear_user) - ! - mov #0, r0 - mov #0xe0, r1 ! 0xffffffe0 - ! - ! r4..(r4+31)&~32 -------- not aligned [ Area 0 ] - ! (r4+31)&~32..(r4+r5)&~32 -------- aligned [ Area 1 ] - ! (r4+r5)&~32..r4+r5 -------- not aligned [ Area 2 ] - ! - ! Clear area 0 - mov r4, r2 - ! - tst r1, r5 ! length < 32 - bt .Larea2 ! skip to remainder - ! - add #31, r2 - and r1, r2 - cmp/eq r4, r2 - bt .Larea1 - mov r2, r3 - sub r4, r3 - mov r3, r7 - mov r4, r2 - ! -.L0: dt r3 -0: mov.b r0, @r2 - bf/s .L0 - add #1, r2 - ! - sub r7, r5 - mov r2, r4 -.Larea1: - mov r4, r3 - add r5, r3 - and r1, r3 - cmp/hi r2, r3 - bf .Larea2 - ! - ! Clear area 1 -#if defined(CONFIG_CPU_SH4) -1: movca.l r0, @r2 -#else -1: mov.l r0, @r2 -#endif - add #4, r2 -2: mov.l r0, @r2 - add #4, r2 -3: mov.l r0, @r2 - add #4, r2 -4: mov.l r0, @r2 - add #4, r2 -5: mov.l r0, @r2 - add #4, r2 -6: mov.l r0, @r2 - add #4, r2 -7: mov.l r0, @r2 - add #4, r2 -8: mov.l r0, @r2 - add #4, r2 - cmp/hi r2, r3 - bt/s 1b - nop - ! - ! Clear area 2 -.Larea2: - mov r4, r3 - add r5, r3 - cmp/hs r3, r2 - bt/s .Ldone - sub r2, r3 -.L2: dt r3 -9: mov.b r0, @r2 - bf/s .L2 - add #1, r2 - ! -.Ldone: rts - mov #0, r0 ! return 0 as normal return - - ! return the number of bytes remained -.Lbad_clear_user: - mov r4, r0 - add r5, r0 - rts - sub r2, r0 - -.section __ex_table,"a" - .align 2 - .long 0b, .Lbad_clear_user - .long 1b, .Lbad_clear_user - .long 2b, .Lbad_clear_user - .long 3b, .Lbad_clear_user - .long 4b, .Lbad_clear_user - .long 5b, .Lbad_clear_user - .long 6b, .Lbad_clear_user - .long 7b, .Lbad_clear_user - .long 8b, .Lbad_clear_user - .long 9b, .Lbad_clear_user -.previous diff --git a/arch/sh/mm/copy_page.S b/arch/sh/mm/copy_page.S deleted file mode 100644 index b879545fa28b..000000000000 --- a/arch/sh/mm/copy_page.S +++ /dev/null @@ -1,388 +0,0 @@ -/* - * copy_page, __copy_user_page, __copy_user implementation of SuperH - * - * Copyright (C) 2001 Niibe Yutaka & Kaz Kojima - * Copyright (C) 2002 Toshinobu Sugioka - * Copyright (C) 2006 Paul Mundt - */ -#include -#include - -/* - * copy_page - * @to: P1 address - * @from: P1 address - * - * void copy_page(void *to, void *from) - */ - -/* - * r0, r1, r2, r3, r4, r5, r6, r7 --- scratch - * r8 --- from + PAGE_SIZE - * r9 --- not used - * r10 --- to - * r11 --- from - */ -ENTRY(copy_page) - mov.l r8,@-r15 - mov.l r10,@-r15 - mov.l r11,@-r15 - mov r4,r10 - mov r5,r11 - mov r5,r8 - mov.l .Lpsz,r0 - add r0,r8 - ! -1: mov.l @r11+,r0 - mov.l @r11+,r1 - mov.l @r11+,r2 - mov.l @r11+,r3 - mov.l @r11+,r4 - mov.l @r11+,r5 - mov.l @r11+,r6 - mov.l @r11+,r7 -#if defined(CONFIG_CPU_SH3) - mov.l r0,@r10 -#elif defined(CONFIG_CPU_SH4) - movca.l r0,@r10 - mov r10,r0 -#endif - add #32,r10 - mov.l r7,@-r10 - mov.l r6,@-r10 - mov.l r5,@-r10 - mov.l r4,@-r10 - mov.l r3,@-r10 - mov.l r2,@-r10 - mov.l r1,@-r10 -#if defined(CONFIG_CPU_SH4) - ocbwb @r0 -#endif - cmp/eq r11,r8 - bf/s 1b - add #28,r10 - ! - mov.l @r15+,r11 - mov.l @r15+,r10 - mov.l @r15+,r8 - rts - nop - - .align 2 -.Lpsz: .long PAGE_SIZE -/* - * __kernel_size_t __copy_user(void *to, const void *from, __kernel_size_t n); - * Return the number of bytes NOT copied - */ -#define EX(...) \ - 9999: __VA_ARGS__ ; \ - .section __ex_table, "a"; \ - .long 9999b, 6000f ; \ - .previous -ENTRY(__copy_user) - ! Check if small number of bytes - mov #11,r0 - mov r4,r3 - cmp/gt r0,r6 ! r6 (len) > r0 (11) - bf/s .L_cleanup_loop_no_pop - add r6,r3 ! last destination address - - ! Calculate bytes needed to align to src - mov.l r11,@-r15 - neg r5,r0 - mov.l r10,@-r15 - add #4,r0 - mov.l r9,@-r15 - and #3,r0 - mov.l r8,@-r15 - tst r0,r0 - bt 2f - -1: - ! Copy bytes to long word align src -EX( mov.b @r5+,r1 ) - dt r0 - add #-1,r6 -EX( mov.b r1,@r4 ) - bf/s 1b - add #1,r4 - - ! Jump to appropriate routine depending on dest -2: mov #3,r1 - mov r6, r2 - and r4,r1 - shlr2 r2 - shll2 r1 - mova .L_jump_tbl,r0 - mov.l @(r0,r1),r1 - jmp @r1 - nop - - .align 2 -.L_jump_tbl: - .long .L_dest00 - .long .L_dest01 - .long .L_dest10 - .long .L_dest11 - -/* - * Come here if there are less than 12 bytes to copy - * - * Keep the branch target close, so the bf/s callee doesn't overflow - * and result in a more expensive branch being inserted. This is the - * fast-path for small copies, the jump via the jump table will hit the - * default slow-path cleanup. -PFM. - */ -.L_cleanup_loop_no_pop: - tst r6,r6 ! Check explicitly for zero - bt 1f - -2: -EX( mov.b @r5+,r0 ) - dt r6 -EX( mov.b r0,@r4 ) - bf/s 2b - add #1,r4 - -1: mov #0,r0 ! normal return -5000: - -# Exception handler: -.section .fixup, "ax" -6000: - mov.l 8000f,r1 - mov r3,r0 - jmp @r1 - sub r4,r0 - .align 2 -8000: .long 5000b - -.previous - rts - nop - -! Destination = 00 - -.L_dest00: - ! Skip the large copy for small transfers - mov #(32+32-4), r0 - cmp/gt r6, r0 ! r0 (60) > r6 (len) - bt 1f - - ! Align dest to a 32 byte boundary - neg r4,r0 - add #0x20, r0 - and #0x1f, r0 - tst r0, r0 - bt 2f - - sub r0, r6 - shlr2 r0 -3: -EX( mov.l @r5+,r1 ) - dt r0 -EX( mov.l r1,@r4 ) - bf/s 3b - add #4,r4 - -2: -EX( mov.l @r5+,r0 ) -EX( mov.l @r5+,r1 ) -EX( mov.l @r5+,r2 ) -EX( mov.l @r5+,r7 ) -EX( mov.l @r5+,r8 ) -EX( mov.l @r5+,r9 ) -EX( mov.l @r5+,r10 ) -EX( mov.l @r5+,r11 ) -#ifdef CONFIG_CPU_SH4 -EX( movca.l r0,@r4 ) -#else -EX( mov.l r0,@r4 ) -#endif - add #-32, r6 -EX( mov.l r1,@(4,r4) ) - mov #32, r0 -EX( mov.l r2,@(8,r4) ) - cmp/gt r6, r0 ! r0 (32) > r6 (len) -EX( mov.l r7,@(12,r4) ) -EX( mov.l r8,@(16,r4) ) -EX( mov.l r9,@(20,r4) ) -EX( mov.l r10,@(24,r4) ) -EX( mov.l r11,@(28,r4) ) - bf/s 2b - add #32,r4 - -1: mov r6, r0 - shlr2 r0 - tst r0, r0 - bt .L_cleanup -1: -EX( mov.l @r5+,r1 ) - dt r0 -EX( mov.l r1,@r4 ) - bf/s 1b - add #4,r4 - - bra .L_cleanup - nop - -! Destination = 10 - -.L_dest10: - mov r2,r7 - shlr2 r7 - shlr r7 - tst r7,r7 - mov #7,r0 - bt/s 1f - and r0,r2 -2: - dt r7 -#ifdef CONFIG_CPU_LITTLE_ENDIAN -EX( mov.l @r5+,r0 ) -EX( mov.l @r5+,r1 ) -EX( mov.l @r5+,r8 ) -EX( mov.l @r5+,r9 ) -EX( mov.l @r5+,r10 ) -EX( mov.w r0,@r4 ) - add #2,r4 - xtrct r1,r0 - xtrct r8,r1 - xtrct r9,r8 - xtrct r10,r9 - -EX( mov.l r0,@r4 ) -EX( mov.l r1,@(4,r4) ) -EX( mov.l r8,@(8,r4) ) -EX( mov.l r9,@(12,r4) ) - -EX( mov.l @r5+,r1 ) -EX( mov.l @r5+,r8 ) -EX( mov.l @r5+,r0 ) - xtrct r1,r10 - xtrct r8,r1 - xtrct r0,r8 - shlr16 r0 -EX( mov.l r10,@(16,r4) ) -EX( mov.l r1,@(20,r4) ) -EX( mov.l r8,@(24,r4) ) -EX( mov.w r0,@(28,r4) ) - bf/s 2b - add #30,r4 -#else -EX( mov.l @(28,r5),r0 ) -EX( mov.l @(24,r5),r8 ) -EX( mov.l @(20,r5),r9 ) -EX( mov.l @(16,r5),r10 ) -EX( mov.w r0,@(30,r4) ) - add #-2,r4 - xtrct r8,r0 - xtrct r9,r8 - xtrct r10,r9 -EX( mov.l r0,@(28,r4) ) -EX( mov.l r8,@(24,r4) ) -EX( mov.l r9,@(20,r4) ) - -EX( mov.l @(12,r5),r0 ) -EX( mov.l @(8,r5),r8 ) - xtrct r0,r10 -EX( mov.l @(4,r5),r9 ) - mov.l r10,@(16,r4) -EX( mov.l @r5,r10 ) - xtrct r8,r0 - xtrct r9,r8 - xtrct r10,r9 -EX( mov.l r0,@(12,r4) ) -EX( mov.l r8,@(8,r4) ) - swap.w r10,r0 -EX( mov.l r9,@(4,r4) ) -EX( mov.w r0,@(2,r4) ) - - add #32,r5 - bf/s 2b - add #34,r4 -#endif - tst r2,r2 - bt .L_cleanup - -1: ! Read longword, write two words per iteration -EX( mov.l @r5+,r0 ) - dt r2 -#ifdef CONFIG_CPU_LITTLE_ENDIAN -EX( mov.w r0,@r4 ) - shlr16 r0 -EX( mov.w r0,@(2,r4) ) -#else -EX( mov.w r0,@(2,r4) ) - shlr16 r0 -EX( mov.w r0,@r4 ) -#endif - bf/s 1b - add #4,r4 - - bra .L_cleanup - nop - -! Destination = 01 or 11 - -.L_dest01: -.L_dest11: - ! Read longword, write byte, word, byte per iteration -EX( mov.l @r5+,r0 ) - dt r2 -#ifdef CONFIG_CPU_LITTLE_ENDIAN -EX( mov.b r0,@r4 ) - shlr8 r0 - add #1,r4 -EX( mov.w r0,@r4 ) - shlr16 r0 -EX( mov.b r0,@(2,r4) ) - bf/s .L_dest01 - add #3,r4 -#else -EX( mov.b r0,@(3,r4) ) - shlr8 r0 - swap.w r0,r7 -EX( mov.b r7,@r4 ) - add #1,r4 -EX( mov.w r0,@r4 ) - bf/s .L_dest01 - add #3,r4 -#endif - -! Cleanup last few bytes -.L_cleanup: - mov r6,r0 - and #3,r0 - tst r0,r0 - bt .L_exit - mov r0,r6 - -.L_cleanup_loop: -EX( mov.b @r5+,r0 ) - dt r6 -EX( mov.b r0,@r4 ) - bf/s .L_cleanup_loop - add #1,r4 - -.L_exit: - mov #0,r0 ! normal return - -5000: - -# Exception handler: -.section .fixup, "ax" -6000: - mov.l 8000f,r1 - mov r3,r0 - jmp @r1 - sub r4,r0 - .align 2 -8000: .long 5000b - -.previous - mov.l @r15+,r8 - mov.l @r15+,r9 - mov.l @r15+,r10 - rts - mov.l @r15+,r11 -- cgit v1.2.3 From eddeeb32fe303910c58c4e3c27fde4b6f1503350 Mon Sep 17 00:00:00 2001 From: Stuart Menefy Date: Mon, 26 Nov 2007 21:32:40 +0900 Subject: sh: Invalidate the TLB after applying PMB mappings. Signed-off-by: Stuart Menefy Signed-off-by: Paul Mundt --- arch/sh/mm/pmb.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/pmb.c b/arch/sh/mm/pmb.c index 1d45b82f0a63..b632051d6ce5 100644 --- a/arch/sh/mm/pmb.c +++ b/arch/sh/mm/pmb.c @@ -27,6 +27,7 @@ #include #include #include +#include #define NR_PMB_ENTRIES 16 @@ -329,6 +330,11 @@ static int __init pmb_init(void) /* PMB.SE and UB[7] */ ctrl_outl((1 << 31) | (1 << 7), PMB_PASCR); + /* Flush out the TLB */ + i = ctrl_inl(MMUCR); + i |= MMUCR_TI; + ctrl_outl(i, MMUCR); + back_to_P1(); return 0; -- cgit v1.2.3 From a23ba43573a24c351640bc19c06c701798fe6e25 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Wed, 28 Nov 2007 20:19:38 +0900 Subject: sh: comment tidying for sh64->sh migration. Signed-off-by: Paul Mundt --- arch/sh/mm/cache-sh5.c | 15 ++++++--------- arch/sh/mm/extable_64.c | 28 +++++++++++++++------------- arch/sh/mm/tlb-sh5.c | 4 +--- 3 files changed, 22 insertions(+), 25 deletions(-) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/cache-sh5.c b/arch/sh/mm/cache-sh5.c index 421487cfff4c..4617e3aeee73 100644 --- a/arch/sh/mm/cache-sh5.c +++ b/arch/sh/mm/cache-sh5.c @@ -1,18 +1,15 @@ /* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * arch/sh64/mm/cache.c + * arch/sh/mm/cache-sh5.c * * Original version Copyright (C) 2000, 2001 Paolo Alberelli * Second version Copyright (C) benedict.gaster@superh.com 2002 * Third version Copyright Richard.Curnow@superh.com 2003 * Hacks to third version Copyright (C) 2003 Paul Mundt + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. */ - -/****************************************************************************/ - #include #include #include @@ -146,7 +143,7 @@ int __init sh64_cache_init(void) /* The following group of functions deal with mapping and unmapping a temporary page into the DTLB slot that have been set aside for our exclusive use. */ /* In order to accomplish this, we use the generic interface for adding and - removing a wired slot entry as defined in arch/sh64/mm/tlb.c */ + removing a wired slot entry as defined in arch/sh/mm/tlb-sh5.c */ /****************************************************************************/ static unsigned long slot_own_flags; diff --git a/arch/sh/mm/extable_64.c b/arch/sh/mm/extable_64.c index a2e6e0563772..f05499688d88 100644 --- a/arch/sh/mm/extable_64.c +++ b/arch/sh/mm/extable_64.c @@ -1,14 +1,14 @@ /* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * arch/sh64/mm/extable.c + * arch/sh/mm/extable_64.c * * Copyright (C) 2003 Richard Curnow * Copyright (C) 2003, 2004 Paul Mundt * * Cloned from the 2.5 SH version.. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. */ #include #include @@ -21,13 +21,16 @@ static const struct exception_table_entry __copy_user_fixup_ex = { .fixup = (unsigned long)&__copy_user_fixup, }; -/* Some functions that may trap due to a bad user-mode address have too many loads - and stores in them to make it at all practical to label each one and put them all in - the main exception table. - - In particular, the fast memcpy routine is like this. It's fix-up is just to fall back - to a slow byte-at-a-time copy, which is handled the conventional way. So it's functionally - OK to just handle any trap occurring in the fast memcpy with that fixup. */ +/* + * Some functions that may trap due to a bad user-mode address have too + * many loads and stores in them to make it at all practical to label + * each one and put them all in the main exception table. + * + * In particular, the fast memcpy routine is like this. It's fix-up is + * just to fall back to a slow byte-at-a-time copy, which is handled the + * conventional way. So it's functionally OK to just handle any trap + * occurring in the fast memcpy with that fixup. + */ static const struct exception_table_entry *check_exception_ranges(unsigned long addr) { if ((addr >= (unsigned long)©_user_memcpy) && @@ -77,4 +80,3 @@ int fixup_exception(struct pt_regs *regs) return 0; } - diff --git a/arch/sh/mm/tlb-sh5.c b/arch/sh/mm/tlb-sh5.c index d517e7d70340..f34274a1ded3 100644 --- a/arch/sh/mm/tlb-sh5.c +++ b/arch/sh/mm/tlb-sh5.c @@ -1,5 +1,5 @@ /* - * arch/sh64/mm/tlb.c + * arch/sh/mm/tlb-sh5.c * * Copyright (C) 2003 Paul Mundt * Copyright (C) 2003 Richard Curnow @@ -7,7 +7,6 @@ * This file is subject to the terms and conditions of the GNU General Public * License. See the file "COPYING" in the main directory of this archive * for more details. - * */ #include #include @@ -163,4 +162,3 @@ inline void sh64_setup_tlb_slot(unsigned long long config_addr, */ inline void sh64_teardown_tlb_slot(unsigned long long config_addr) __attribute__ ((alias("__flush_tlb_slot"))); - -- cgit v1.2.3 From 53ff09422e5e7a6d6198b767c8f494e43ec8e3ae Mon Sep 17 00:00:00 2001 From: Nobuhiro Iwamatsu Date: Fri, 30 Nov 2007 12:33:17 +0900 Subject: sh: Fix compile error of arch/sh/mm/pmb.c Signed-off-by: Nobuhiro Iwamatsu Signed-off-by: Paul Mundt --- arch/sh/mm/pmb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/pmb.c b/arch/sh/mm/pmb.c index b632051d6ce5..ef6ab39eaf65 100644 --- a/arch/sh/mm/pmb.c +++ b/arch/sh/mm/pmb.c @@ -305,7 +305,7 @@ static void pmb_cache_ctor(struct kmem_cache *cachep, void *pmb) static int __init pmb_init(void) { unsigned int nr_entries = ARRAY_SIZE(pmb_init_map); - unsigned int entry; + unsigned int entry, i; BUG_ON(unlikely(nr_entries >= NR_PMB_ENTRIES)); -- cgit v1.2.3 From cbaa118ecfd99fc5ed7adbd9c34a30e1c05e3c93 Mon Sep 17 00:00:00 2001 From: Stuart Menefy Date: Fri, 30 Nov 2007 17:06:36 +0900 Subject: sh: Preparation for uncached jumps through PMB. Presently most of the 29-bit physical parts do P1/P2 segmentation with a 1:1 cached/uncached mapping, jumping between the two to control the caching behaviour. This provides the basic infrastructure to maintain this behaviour on 32-bit physical parts that don't map P1/P2 at all, using a shiny new linker section and corresponding fixmap entry. Signed-off-by: Stuart Menefy Signed-off-by: Paul Mundt --- arch/sh/mm/cache-debugfs.c | 9 +++++---- arch/sh/mm/cache-sh4.c | 14 +++++++------- arch/sh/mm/cache-sh7705.c | 12 ++++++------ arch/sh/mm/init.c | 16 +++++++++++++++- arch/sh/mm/pmb.c | 18 +++++++++--------- arch/sh/mm/tlb-sh4.c | 7 ++++--- 6 files changed, 46 insertions(+), 30 deletions(-) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/cache-debugfs.c b/arch/sh/mm/cache-debugfs.c index de6d2c9aa477..db6d950b6f5e 100644 --- a/arch/sh/mm/cache-debugfs.c +++ b/arch/sh/mm/cache-debugfs.c @@ -22,7 +22,8 @@ enum cache_type { CACHE_TYPE_UNIFIED, }; -static int cache_seq_show(struct seq_file *file, void *iter) +static int __uses_jump_to_uncached cache_seq_show(struct seq_file *file, + void *iter) { unsigned int cache_type = (unsigned int)file->private; struct cache_info *cache; @@ -34,11 +35,11 @@ static int cache_seq_show(struct seq_file *file, void *iter) * Go uncached immediately so we don't skew the results any * more than we already are.. */ - jump_to_P2(); + jump_to_uncached(); ccr = ctrl_inl(CCR); if ((ccr & CCR_CACHE_ENABLE) == 0) { - back_to_P1(); + back_to_cached(); seq_printf(file, "disabled\n"); return 0; @@ -104,7 +105,7 @@ static int cache_seq_show(struct seq_file *file, void *iter) addrstart += cache->way_incr; } - back_to_P1(); + back_to_cached(); return 0; } diff --git a/arch/sh/mm/cache-sh4.c b/arch/sh/mm/cache-sh4.c index 226b190c5b9c..43d7ff6b6ec7 100644 --- a/arch/sh/mm/cache-sh4.c +++ b/arch/sh/mm/cache-sh4.c @@ -190,7 +190,7 @@ void flush_icache_range(unsigned long start, unsigned long end) * .. which happens to be the same behavior as flush_icache_range(). * So, we simply flush out a line. */ -void flush_cache_sigtramp(unsigned long addr) +void __uses_jump_to_uncached flush_cache_sigtramp(unsigned long addr) { unsigned long v, index; unsigned long flags; @@ -205,13 +205,13 @@ void flush_cache_sigtramp(unsigned long addr) (v & boot_cpu_data.icache.entry_mask); local_irq_save(flags); - jump_to_P2(); + jump_to_uncached(); for (i = 0; i < boot_cpu_data.icache.ways; i++, index += boot_cpu_data.icache.way_incr) ctrl_outl(0, index); /* Clear out Valid-bit */ - back_to_P1(); + back_to_cached(); wmb(); local_irq_restore(flags); } @@ -256,12 +256,12 @@ void flush_dcache_page(struct page *page) } /* TODO: Selective icache invalidation through IC address array.. */ -static inline void flush_icache_all(void) +static inline void __uses_jump_to_uncached flush_icache_all(void) { unsigned long flags, ccr; local_irq_save(flags); - jump_to_P2(); + jump_to_uncached(); /* Flush I-cache */ ccr = ctrl_inl(CCR); @@ -269,11 +269,11 @@ static inline void flush_icache_all(void) ctrl_outl(ccr, CCR); /* - * back_to_P1() will take care of the barrier for us, don't add + * back_to_cached() will take care of the barrier for us, don't add * another one! */ - back_to_P1(); + back_to_cached(); local_irq_restore(flags); } diff --git a/arch/sh/mm/cache-sh7705.c b/arch/sh/mm/cache-sh7705.c index 4896d7376926..22dacc778823 100644 --- a/arch/sh/mm/cache-sh7705.c +++ b/arch/sh/mm/cache-sh7705.c @@ -71,7 +71,7 @@ void flush_icache_range(unsigned long start, unsigned long end) /* * Writeback&Invalidate the D-cache of the page */ -static void __flush_dcache_page(unsigned long phys) +static void __uses_jump_to_uncached __flush_dcache_page(unsigned long phys) { unsigned long ways, waysize, addrstart; unsigned long flags; @@ -92,7 +92,7 @@ static void __flush_dcache_page(unsigned long phys) * possible. */ local_irq_save(flags); - jump_to_P2(); + jump_to_uncached(); ways = current_cpu_data.dcache.ways; waysize = current_cpu_data.dcache.sets; @@ -118,7 +118,7 @@ static void __flush_dcache_page(unsigned long phys) addrstart += current_cpu_data.dcache.way_incr; } while (--ways); - back_to_P1(); + back_to_cached(); local_irq_restore(flags); } @@ -132,15 +132,15 @@ void flush_dcache_page(struct page *page) __flush_dcache_page(PHYSADDR(page_address(page))); } -void flush_cache_all(void) +void __uses_jump_to_uncached flush_cache_all(void) { unsigned long flags; local_irq_save(flags); - jump_to_P2(); + jump_to_uncached(); cache_wback_all(); - back_to_P1(); + back_to_cached(); local_irq_restore(flags); } diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index 79c309780f95..094225e0d722 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -23,6 +23,7 @@ DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); pgd_t swapper_pg_dir[PTRS_PER_PGD]; +unsigned long cached_to_uncached = 0; void show_mem(void) { @@ -99,7 +100,8 @@ static void set_pte_phys(unsigned long addr, unsigned long phys, pgprot_t prot) set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, prot)); - flush_tlb_one(get_asid(), addr); + if (cached_to_uncached) + flush_tlb_one(get_asid(), addr); } /* @@ -164,6 +166,18 @@ void __init paging_init(void) } free_area_init_nodes(max_zone_pfns); + + /* Set up the uncached fixmap */ + set_fixmap_nocache(FIX_UNCACHED, __pa(&__uncached_start)); + +#ifdef CONFIG_29BIT + /* + * Handle trivial transitions between cached and uncached + * segments, making use of the 1:1 mapping relationship in + * 512MB lowmem. + */ + cached_to_uncached = P2SEG - P1SEG; +#endif } static struct kcore_list kcore_mem, kcore_vmalloc; diff --git a/arch/sh/mm/pmb.c b/arch/sh/mm/pmb.c index ef6ab39eaf65..ab81c602295f 100644 --- a/arch/sh/mm/pmb.c +++ b/arch/sh/mm/pmb.c @@ -163,18 +163,18 @@ repeat: return 0; } -int set_pmb_entry(struct pmb_entry *pmbe) +int __uses_jump_to_uncached set_pmb_entry(struct pmb_entry *pmbe) { int ret; - jump_to_P2(); + jump_to_uncached(); ret = __set_pmb_entry(pmbe->vpn, pmbe->ppn, pmbe->flags, &pmbe->entry); - back_to_P1(); + back_to_cached(); return ret; } -void clear_pmb_entry(struct pmb_entry *pmbe) +void __uses_jump_to_uncached clear_pmb_entry(struct pmb_entry *pmbe) { unsigned int entry = pmbe->entry; unsigned long addr; @@ -188,7 +188,7 @@ void clear_pmb_entry(struct pmb_entry *pmbe) entry >= NR_PMB_ENTRIES)) return; - jump_to_P2(); + jump_to_uncached(); /* Clear V-bit */ addr = mk_pmb_addr(entry); @@ -197,7 +197,7 @@ void clear_pmb_entry(struct pmb_entry *pmbe) addr = mk_pmb_data(entry); ctrl_outl(ctrl_inl(addr) & ~PMB_V, addr); - back_to_P1(); + back_to_cached(); clear_bit(entry, &pmb_map); } @@ -302,7 +302,7 @@ static void pmb_cache_ctor(struct kmem_cache *cachep, void *pmb) pmbe->entry = PMB_NO_ENTRY; } -static int __init pmb_init(void) +static int __uses_jump_to_uncached pmb_init(void) { unsigned int nr_entries = ARRAY_SIZE(pmb_init_map); unsigned int entry, i; @@ -312,7 +312,7 @@ static int __init pmb_init(void) pmb_cache = kmem_cache_create("pmb", sizeof(struct pmb_entry), 0, SLAB_PANIC, pmb_cache_ctor); - jump_to_P2(); + jump_to_uncached(); /* * Ordering is important, P2 must be mapped in the PMB before we @@ -335,7 +335,7 @@ static int __init pmb_init(void) i |= MMUCR_TI; ctrl_outl(i, MMUCR); - back_to_P1(); + back_to_cached(); return 0; } diff --git a/arch/sh/mm/tlb-sh4.c b/arch/sh/mm/tlb-sh4.c index 2d1dd6044307..f0c7b7397fa6 100644 --- a/arch/sh/mm/tlb-sh4.c +++ b/arch/sh/mm/tlb-sh4.c @@ -79,7 +79,8 @@ void update_mmu_cache(struct vm_area_struct * vma, local_irq_restore(flags); } -void local_flush_tlb_one(unsigned long asid, unsigned long page) +void __uses_jump_to_uncached local_flush_tlb_one(unsigned long asid, + unsigned long page) { unsigned long addr, data; @@ -91,7 +92,7 @@ void local_flush_tlb_one(unsigned long asid, unsigned long page) */ addr = MMU_UTLB_ADDRESS_ARRAY | MMU_PAGE_ASSOC_BIT; data = page | asid; /* VALID bit is off */ - jump_to_P2(); + jump_to_uncached(); ctrl_outl(data, addr); - back_to_P1(); + back_to_cached(); } -- cgit v1.2.3 From 2adb4e10092ee1719539bcfe397b96dbdc863e2e Mon Sep 17 00:00:00 2001 From: Stuart Menefy Date: Fri, 30 Nov 2007 17:59:55 +0900 Subject: sh: Populate swapper_pg_dir with fixmap range. This saves us from having to use kmalloc() for the fixmap entries, which is needed early for the uncached fixmap. Signed-off-by: Stuart Menefy Signed-off-by: Paul Mundt --- arch/sh/mm/init.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index 094225e0d722..2918c6b14659 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -130,6 +130,37 @@ void __set_fixmap(enum fixed_addresses idx, unsigned long phys, pgprot_t prot) set_pte_phys(address, phys, prot); } + +void __init page_table_range_init(unsigned long start, unsigned long end, + pgd_t *pgd_base) +{ + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + int pgd_idx; + unsigned long vaddr; + + vaddr = start & PMD_MASK; + end = (end + PMD_SIZE - 1) & PMD_MASK; + pgd_idx = pgd_index(vaddr); + pgd = pgd_base + pgd_idx; + + for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) { + BUG_ON(pgd_none(*pgd)); + pud = pud_offset(pgd, 0); + BUG_ON(pud_none(*pud)); + pmd = pmd_offset(pud, 0); + + if (!pmd_present(*pmd)) { + pte_t *pte_table; + pte_table = (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE); + memset(pte_table, 0, PAGE_SIZE); + pmd_populate_kernel(&init_mm, pmd, pte_table); + } + + vaddr += PMD_SIZE; + } +} #endif /* CONFIG_MMU */ /* @@ -149,6 +180,11 @@ void __init paging_init(void) * check for a null value. */ set_TTB(swapper_pg_dir); + /* Populate the relevant portions of swapper_pg_dir so that + * we can use the fixmap entries without calling kmalloc. + * pte's will be filled in by __set_fixmap(). */ + page_table_range_init(FIXADDR_START, FIXADDR_TOP, swapper_pg_dir); + memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); for_each_online_node(nid) { -- cgit v1.2.3 From 9acb98fb7ce948063a2269b4f8db83d6bef7e2b0 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Mon, 17 Dec 2007 10:52:11 +0900 Subject: sh: Stub in page_table_range_init() on nommu. Signed-off-by: Paul Mundt --- arch/sh/mm/tlb-nommu.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/tlb-nommu.c b/arch/sh/mm/tlb-nommu.c index 1ccca7c0532e..8842620604c3 100644 --- a/arch/sh/mm/tlb-nommu.c +++ b/arch/sh/mm/tlb-nommu.c @@ -9,6 +9,7 @@ */ #include #include +#include /* * Nothing too terribly exciting here .. @@ -49,3 +50,8 @@ void update_mmu_cache(struct vm_area_struct * vma, { BUG(); } + +void __init page_table_range_init(unsigned long start, unsigned long end, + pgd_t *pgd_base) +{ +} -- cgit v1.2.3 From 52e8b118ecd17185ce514cd3f955094c1d8f4288 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Tue, 18 Dec 2007 09:43:35 +0900 Subject: sh: Provide a stubbed __set_fixmap() for nommu. Signed-off-by: Paul Mundt --- arch/sh/mm/tlb-nommu.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/tlb-nommu.c b/arch/sh/mm/tlb-nommu.c index 8842620604c3..15111bc7ddd6 100644 --- a/arch/sh/mm/tlb-nommu.c +++ b/arch/sh/mm/tlb-nommu.c @@ -55,3 +55,7 @@ void __init page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base) { } + +void __set_fixmap(enum fixed_addresses idx, unsigned long phys, pgprot_t prot) +{ +} -- cgit v1.2.3 From b62ad83d91ebf1368e9e72d476b18698ac67bef9 Mon Sep 17 00:00:00 2001 From: Paul Mundt Date: Thu, 10 Jan 2008 14:07:03 +0900 Subject: sh: Correct pte size mismatch for X2 TLB. Fixes up a build warning/error in arch/sh/mm/fault_32.c. Signed-off-by: Paul Mundt --- arch/sh/mm/fault_32.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/fault_32.c b/arch/sh/mm/fault_32.c index 60d74f793a1d..33b43d20e9f6 100644 --- a/arch/sh/mm/fault_32.c +++ b/arch/sh/mm/fault_32.c @@ -172,7 +172,7 @@ no_context: bust_spinlocks(1); if (oops_may_print()) { - __typeof__(pte_val(__pte(0))) page; + unsigned long page; if (address < PAGE_SIZE) printk(KERN_ALERT "Unable to handle kernel NULL " -- cgit v1.2.3 From f93e97eaead5c50af35d73cca7301ebbfdff116c Mon Sep 17 00:00:00 2001 From: Magnus Damm Date: Thu, 24 Jan 2008 18:35:10 +0900 Subject: sh: declared coherent memory support V2 This patch adds declared coherent memory support to the sh architecture. All functions are based on the x86 implementation. Header files are adjusted to use the new functions instead of the former consistent_alloc() code. This version includes the few changes what were included in the fix patch together with modifications based on feedback from Paul. Signed-off-by: Magnus Damm Signed-off-by: Paul Mundt --- arch/sh/mm/consistent.c | 174 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 128 insertions(+), 46 deletions(-) (limited to 'arch/sh/mm') diff --git a/arch/sh/mm/consistent.c b/arch/sh/mm/consistent.c index 65ad30031ad7..7b2131c9eeda 100644 --- a/arch/sh/mm/consistent.c +++ b/arch/sh/mm/consistent.c @@ -3,6 +3,8 @@ * * Copyright (C) 2004 - 2007 Paul Mundt * + * Declared coherent memory functions based on arch/x86/kernel/pci-dma_32.c + * * This file is subject to the terms and conditions of the GNU General Public * License. See the file "COPYING" in the main directory of this archive * for more details. @@ -13,66 +15,146 @@ #include #include -void *consistent_alloc(gfp_t gfp, size_t size, dma_addr_t *handle) -{ - struct page *page, *end, *free; - void *ret, *vp; - int order; - - size = PAGE_ALIGN(size); - order = get_order(size); +struct dma_coherent_mem { + void *virt_base; + u32 device_base; + int size; + int flags; + unsigned long *bitmap; +}; - page = alloc_pages(gfp, order); - if (!page) - return NULL; - split_page(page, order); +void *dma_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp) +{ + void *ret; + struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL; + int order = get_order(size); + + if (mem) { + int page = bitmap_find_free_region(mem->bitmap, mem->size, + order); + if (page >= 0) { + *dma_handle = mem->device_base + (page << PAGE_SHIFT); + ret = mem->virt_base + (page << PAGE_SHIFT); + memset(ret, 0, size); + return ret; + } + if (mem->flags & DMA_MEMORY_EXCLUSIVE) + return NULL; + } - ret = page_address(page); - *handle = virt_to_phys(ret); + ret = (void *)__get_free_pages(gfp, order); - vp = ioremap_nocache(*handle, size); - if (!vp) { - free_pages((unsigned long)ret, order); - return NULL; + if (ret != NULL) { + memset(ret, 0, size); + /* + * Pages from the page allocator may have data present in + * cache. So flush the cache before using uncached memory. + */ + dma_cache_sync(NULL, ret, size, DMA_BIDIRECTIONAL); + *dma_handle = virt_to_phys(ret); } + return ret; +} +EXPORT_SYMBOL(dma_alloc_coherent); - memset(vp, 0, size); - - /* - * We must flush the cache before we pass it on to the device - */ - dma_cache_sync(NULL, ret, size, DMA_BIDIRECTIONAL); +void dma_free_coherent(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle) +{ + struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL; + int order = get_order(size); - page = virt_to_page(ret); - free = page + (size >> PAGE_SHIFT); - end = page + (1 << order); + if (mem && vaddr >= mem->virt_base && vaddr < (mem->virt_base + (mem->size << PAGE_SHIFT))) { + int page = (vaddr - mem->virt_base) >> PAGE_SHIFT; - while (++page < end) { - /* Free any unused pages */ - if (page >= free) { - __free_page(page); - } + bitmap_release_region(mem->bitmap, page, order); + } else { + WARN_ON(irqs_disabled()); /* for portability */ + BUG_ON(mem && mem->flags & DMA_MEMORY_EXCLUSIVE); + free_pages((unsigned long)vaddr, order); } - - return vp; } -EXPORT_SYMBOL(consistent_alloc); +EXPORT_SYMBOL(dma_free_coherent); -void consistent_free(void *vaddr, size_t size, dma_addr_t dma_handle) +int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr, + dma_addr_t device_addr, size_t size, int flags) { - struct page *page; - unsigned long addr; - - addr = (unsigned long)phys_to_virt((unsigned long)dma_handle); - page = virt_to_page(addr); + void __iomem *mem_base = NULL; + int pages = size >> PAGE_SHIFT; + int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long); + + if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0) + goto out; + if (!size) + goto out; + if (dev->dma_mem) + goto out; + + /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */ + + mem_base = ioremap_nocache(bus_addr, size); + if (!mem_base) + goto out; + + dev->dma_mem = kmalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL); + if (!dev->dma_mem) + goto out; + dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL); + if (!dev->dma_mem->bitmap) + goto free1_out; + + dev->dma_mem->virt_base = mem_base; + dev->dma_mem->device_base = device_addr; + dev->dma_mem->size = pages; + dev->dma_mem->flags = flags; + + if (flags & DMA_MEMORY_MAP) + return DMA_MEMORY_MAP; + + return DMA_MEMORY_IO; + + free1_out: + kfree(dev->dma_mem); + out: + if (mem_base) + iounmap(mem_base); + return 0; +} +EXPORT_SYMBOL(dma_declare_coherent_memory); - free_pages(addr, get_order(size)); +void dma_release_declared_memory(struct device *dev) +{ + struct dma_coherent_mem *mem = dev->dma_mem; + + if (!mem) + return; + dev->dma_mem = NULL; + iounmap(mem->virt_base); + kfree(mem->bitmap); + kfree(mem); +} +EXPORT_SYMBOL(dma_release_declared_memory); - iounmap(vaddr); +void *dma_mark_declared_memory_occupied(struct device *dev, + dma_addr_t device_addr, size_t size) +{ + struct dma_coherent_mem *mem = dev->dma_mem; + int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1) >> PAGE_SHIFT; + int pos, err; + + if (!mem) + return ERR_PTR(-EINVAL); + + pos = (device_addr - mem->device_base) >> PAGE_SHIFT; + err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages)); + if (err != 0) + return ERR_PTR(err); + return mem->virt_base + (pos << PAGE_SHIFT); } -EXPORT_SYMBOL(consistent_free); +EXPORT_SYMBOL(dma_mark_declared_memory_occupied); -void consistent_sync(void *vaddr, size_t size, int direction) +void dma_cache_sync(struct device *dev, void *vaddr, size_t size, + enum dma_data_direction direction) { #ifdef CONFIG_CPU_SH5 void *p1addr = vaddr; @@ -94,4 +176,4 @@ void consistent_sync(void *vaddr, size_t size, int direction) BUG(); } } -EXPORT_SYMBOL(consistent_sync); +EXPORT_SYMBOL(dma_cache_sync); -- cgit v1.2.3