summaryrefslogtreecommitdiff
path: root/arch/arm/mm
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2010-03-01 09:15:15 -0800
committerLinus Torvalds <torvalds@linux-foundation.org>2010-03-01 09:15:15 -0800
commitac0f6f927db539e03e1f3f61bcd4ed57d5cde7a9 (patch)
tree816e5ac643b15c2050c64a7075f0f7e13d86ea09 /arch/arm/mm
parentb1bf9368407ae7e89d8a005bb40beb70a41df539 (diff)
parent9f33be2c3a80bdc2cc08342dd77fac87652e0548 (diff)
Merge branch 'for-linus' of master.kernel.org:/home/rmk/linux-2.6-arm
* 'for-linus' of master.kernel.org:/home/rmk/linux-2.6-arm: (100 commits) ARM: Eliminate decompressor -Dstatic= PIC hack ARM: 5958/1: ARM: U300: fix inverted clk round rate ARM: 5956/1: misplaced parentheses ARM: 5955/1: ep93xx: move timer defines into core.c and document ARM: 5954/1: ep93xx: move gpio interrupt support to gpio.c ARM: 5953/1: ep93xx: fix broken build of clock.c ARM: 5952/1: ARM: MM: Add ARM_L1_CACHE_SHIFT_6 for handle inside each ARCH Kconfig ARM: 5949/1: NUC900 add gpio virtual memory map ARM: 5948/1: Enable timer0 to time4 clock support for nuc910 ARM: 5940/2: ARM: MMCI: remove custom DBG macro and printk ARM: make_coherent(): fix problems with highpte, part 2 MM: Pass a PTE pointer to update_mmu_cache() rather than the PTE itself ARM: 5945/1: ep93xx: include correct irq.h in core.c ARM: 5933/1: amba-pl011: support hardware flow control ARM: 5930/1: Add PKMAP area description to memory.txt. ARM: 5929/1: Add checks to detect overlap of memory regions. ARM: 5928/1: Change type of VMALLOC_END to unsigned long. ARM: 5927/1: Make delimiters of DMA area globally visibly. ARM: 5926/1: Add "Virtual kernel memory..." printout. ARM: 5920/1: OMAP4: Enable L2 Cache ... Fix up trivial conflict in arch/arm/mach-mx25/clock.c
Diffstat (limited to 'arch/arm/mm')
-rw-r--r--arch/arm/mm/Kconfig8
-rw-r--r--arch/arm/mm/alignment.c6
-rw-r--r--arch/arm/mm/cache-fa.S32
-rw-r--r--arch/arm/mm/cache-l2x0.c72
-rw-r--r--arch/arm/mm/cache-v3.S43
-rw-r--r--arch/arm/mm/cache-v4.S43
-rw-r--r--arch/arm/mm/cache-v4wb.S32
-rw-r--r--arch/arm/mm/cache-v4wt.S40
-rw-r--r--arch/arm/mm/cache-v6.S34
-rw-r--r--arch/arm/mm/cache-v7.S34
-rw-r--r--arch/arm/mm/context.c124
-rw-r--r--arch/arm/mm/copypage-feroceon.c3
-rw-r--r--arch/arm/mm/copypage-v3.c2
-rw-r--r--arch/arm/mm/copypage-v4mc.c2
-rw-r--r--arch/arm/mm/copypage-v4wb.c3
-rw-r--r--arch/arm/mm/copypage-v4wt.c2
-rw-r--r--arch/arm/mm/copypage-v6.c4
-rw-r--r--arch/arm/mm/copypage-xsc3.c3
-rw-r--r--arch/arm/mm/copypage-xscale.c2
-rw-r--r--arch/arm/mm/dma-mapping.c162
-rw-r--r--arch/arm/mm/fault-armv.c85
-rw-r--r--arch/arm/mm/fault.c7
-rw-r--r--arch/arm/mm/flush.c51
-rw-r--r--arch/arm/mm/init.c113
-rw-r--r--arch/arm/mm/ioremap.c57
-rw-r--r--arch/arm/mm/mmu.c41
-rw-r--r--arch/arm/mm/nommu.c12
-rw-r--r--arch/arm/mm/proc-arm1020.S32
-rw-r--r--arch/arm/mm/proc-arm1020e.S32
-rw-r--r--arch/arm/mm/proc-arm1022.S32
-rw-r--r--arch/arm/mm/proc-arm1026.S32
-rw-r--r--arch/arm/mm/proc-arm920.S32
-rw-r--r--arch/arm/mm/proc-arm922.S32
-rw-r--r--arch/arm/mm/proc-arm925.S32
-rw-r--r--arch/arm/mm/proc-arm926.S32
-rw-r--r--arch/arm/mm/proc-arm940.S32
-rw-r--r--arch/arm/mm/proc-arm946.S32
-rw-r--r--arch/arm/mm/proc-feroceon.S54
-rw-r--r--arch/arm/mm/proc-mohawk.S32
-rw-r--r--arch/arm/mm/proc-xsc3.S32
-rw-r--r--arch/arm/mm/proc-xscale.S49
41 files changed, 1143 insertions, 361 deletions
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index baf638487a2d..c4ed9f93f646 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -399,7 +399,7 @@ config CPU_V6
config CPU_32v6K
bool "Support ARM V6K processor extensions" if !SMP
depends on CPU_V6
- default y if SMP && !ARCH_MX3
+ default y if SMP && !(ARCH_MX3 || ARCH_OMAP2)
help
Say Y here if your ARMv6 processor supports the 'K' extension.
This enables the kernel to use some instructions not present
@@ -410,7 +410,7 @@ config CPU_32v6K
# ARMv7
config CPU_V7
bool "Support ARM V7 processor" if ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX
- select CPU_32v6K
+ select CPU_32v6K if !ARCH_OMAP2
select CPU_32v7
select CPU_ABRT_EV7
select CPU_PABRT_V7
@@ -754,7 +754,7 @@ config CACHE_FEROCEON_L2_WRITETHROUGH
config CACHE_L2X0
bool "Enable the L2x0 outer cache controller"
depends on REALVIEW_EB_ARM11MP || MACH_REALVIEW_PB11MP || MACH_REALVIEW_PB1176 || \
- REALVIEW_EB_A9MP || ARCH_MX35 || ARCH_MX31 || MACH_REALVIEW_PBX || ARCH_NOMADIK
+ REALVIEW_EB_A9MP || ARCH_MX35 || ARCH_MX31 || MACH_REALVIEW_PBX || ARCH_NOMADIK || ARCH_OMAP4
default y
select OUTER_CACHE
help
@@ -779,5 +779,5 @@ config CACHE_XSC3L2
config ARM_L1_CACHE_SHIFT
int
- default 6 if ARCH_OMAP3 || ARCH_S5PC1XX
+ default 6 if ARM_L1_CACHE_SHIFT_6
default 5
diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c
index 62820eda84d9..edddd66faac6 100644
--- a/arch/arm/mm/alignment.c
+++ b/arch/arm/mm/alignment.c
@@ -901,11 +901,7 @@ static int __init alignment_init(void)
#ifdef CONFIG_PROC_FS
struct proc_dir_entry *res;
- res = proc_mkdir("cpu", NULL);
- if (!res)
- return -ENOMEM;
-
- res = create_proc_entry("alignment", S_IWUSR | S_IRUGO, res);
+ res = create_proc_entry("cpu/alignment", S_IWUSR | S_IRUGO, NULL);
if (!res)
return -ENOMEM;
diff --git a/arch/arm/mm/cache-fa.S b/arch/arm/mm/cache-fa.S
index a89444a3c016..7148e53e6078 100644
--- a/arch/arm/mm/cache-fa.S
+++ b/arch/arm/mm/cache-fa.S
@@ -157,7 +157,7 @@ ENTRY(fa_flush_kern_dcache_area)
* - start - virtual start address
* - end - virtual end address
*/
-ENTRY(fa_dma_inv_range)
+fa_dma_inv_range:
tst r0, #CACHE_DLINESIZE - 1
bic r0, r0, #CACHE_DLINESIZE - 1
mcrne p15, 0, r0, c7, c14, 1 @ clean & invalidate D entry
@@ -180,7 +180,7 @@ ENTRY(fa_dma_inv_range)
* - start - virtual start address
* - end - virtual end address
*/
-ENTRY(fa_dma_clean_range)
+fa_dma_clean_range:
bic r0, r0, #CACHE_DLINESIZE - 1
1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry
add r0, r0, #CACHE_DLINESIZE
@@ -205,6 +205,30 @@ ENTRY(fa_dma_flush_range)
mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(fa_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq fa_dma_clean_range
+ bcs fa_dma_inv_range
+ b fa_dma_flush_range
+ENDPROC(fa_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(fa_dma_unmap_area)
+ mov pc, lr
+ENDPROC(fa_dma_unmap_area)
+
__INITDATA
.type fa_cache_fns, #object
@@ -215,7 +239,7 @@ ENTRY(fa_cache_fns)
.long fa_coherent_kern_range
.long fa_coherent_user_range
.long fa_flush_kern_dcache_area
- .long fa_dma_inv_range
- .long fa_dma_clean_range
+ .long fa_dma_map_area
+ .long fa_dma_unmap_area
.long fa_dma_flush_range
.size fa_cache_fns, . - fa_cache_fns
diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c
index cb8fc6573b1b..07334632d3e2 100644
--- a/arch/arm/mm/cache-l2x0.c
+++ b/arch/arm/mm/cache-l2x0.c
@@ -42,6 +42,57 @@ static inline void cache_sync(void)
cache_wait(base + L2X0_CACHE_SYNC, 1);
}
+static inline void l2x0_clean_line(unsigned long addr)
+{
+ void __iomem *base = l2x0_base;
+ cache_wait(base + L2X0_CLEAN_LINE_PA, 1);
+ writel(addr, base + L2X0_CLEAN_LINE_PA);
+}
+
+static inline void l2x0_inv_line(unsigned long addr)
+{
+ void __iomem *base = l2x0_base;
+ cache_wait(base + L2X0_INV_LINE_PA, 1);
+ writel(addr, base + L2X0_INV_LINE_PA);
+}
+
+#ifdef CONFIG_PL310_ERRATA_588369
+static void debug_writel(unsigned long val)
+{
+ extern void omap_smc1(u32 fn, u32 arg);
+
+ /*
+ * Texas Instrument secure monitor api to modify the
+ * PL310 Debug Control Register.
+ */
+ omap_smc1(0x100, val);
+}
+
+static inline void l2x0_flush_line(unsigned long addr)
+{
+ void __iomem *base = l2x0_base;
+
+ /* Clean by PA followed by Invalidate by PA */
+ cache_wait(base + L2X0_CLEAN_LINE_PA, 1);
+ writel(addr, base + L2X0_CLEAN_LINE_PA);
+ cache_wait(base + L2X0_INV_LINE_PA, 1);
+ writel(addr, base + L2X0_INV_LINE_PA);
+}
+#else
+
+/* Optimised out for non-errata case */
+static inline void debug_writel(unsigned long val)
+{
+}
+
+static inline void l2x0_flush_line(unsigned long addr)
+{
+ void __iomem *base = l2x0_base;
+ cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
+ writel(addr, base + L2X0_CLEAN_INV_LINE_PA);
+}
+#endif
+
static inline void l2x0_inv_all(void)
{
unsigned long flags;
@@ -62,23 +113,24 @@ static void l2x0_inv_range(unsigned long start, unsigned long end)
spin_lock_irqsave(&l2x0_lock, flags);
if (start & (CACHE_LINE_SIZE - 1)) {
start &= ~(CACHE_LINE_SIZE - 1);
- cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
- writel(start, base + L2X0_CLEAN_INV_LINE_PA);
+ debug_writel(0x03);
+ l2x0_flush_line(start);
+ debug_writel(0x00);
start += CACHE_LINE_SIZE;
}
if (end & (CACHE_LINE_SIZE - 1)) {
end &= ~(CACHE_LINE_SIZE - 1);
- cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
- writel(end, base + L2X0_CLEAN_INV_LINE_PA);
+ debug_writel(0x03);
+ l2x0_flush_line(end);
+ debug_writel(0x00);
}
while (start < end) {
unsigned long blk_end = start + min(end - start, 4096UL);
while (start < blk_end) {
- cache_wait(base + L2X0_INV_LINE_PA, 1);
- writel(start, base + L2X0_INV_LINE_PA);
+ l2x0_inv_line(start);
start += CACHE_LINE_SIZE;
}
@@ -103,8 +155,7 @@ static void l2x0_clean_range(unsigned long start, unsigned long end)
unsigned long blk_end = start + min(end - start, 4096UL);
while (start < blk_end) {
- cache_wait(base + L2X0_CLEAN_LINE_PA, 1);
- writel(start, base + L2X0_CLEAN_LINE_PA);
+ l2x0_clean_line(start);
start += CACHE_LINE_SIZE;
}
@@ -128,11 +179,12 @@ static void l2x0_flush_range(unsigned long start, unsigned long end)
while (start < end) {
unsigned long blk_end = start + min(end - start, 4096UL);
+ debug_writel(0x03);
while (start < blk_end) {
- cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1);
- writel(start, base + L2X0_CLEAN_INV_LINE_PA);
+ l2x0_flush_line(start);
start += CACHE_LINE_SIZE;
}
+ debug_writel(0x00);
if (blk_end < end) {
spin_unlock_irqrestore(&l2x0_lock, flags);
diff --git a/arch/arm/mm/cache-v3.S b/arch/arm/mm/cache-v3.S
index 2a482731ea36..c2ff3c599fee 100644
--- a/arch/arm/mm/cache-v3.S
+++ b/arch/arm/mm/cache-v3.S
@@ -84,20 +84,6 @@ ENTRY(v3_flush_kern_dcache_area)
/* FALLTHROUGH */
/*
- * dma_inv_range(start, end)
- *
- * Invalidate (discard) the specified virtual address range.
- * May not write back any entries. If 'start' or 'end'
- * are not cache line aligned, those lines must be written
- * back.
- *
- * - start - virtual start address
- * - end - virtual end address
- */
-ENTRY(v3_dma_inv_range)
- /* FALLTHROUGH */
-
-/*
* dma_flush_range(start, end)
*
* Clean and invalidate the specified virtual address range.
@@ -108,18 +94,29 @@ ENTRY(v3_dma_inv_range)
ENTRY(v3_dma_flush_range)
mov r0, #0
mcr p15, 0, r0, c7, c0, 0 @ flush ID cache
+ mov pc, lr
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(v3_dma_unmap_area)
+ teq r2, #DMA_TO_DEVICE
+ bne v3_dma_flush_range
/* FALLTHROUGH */
/*
- * dma_clean_range(start, end)
- *
- * Clean (write back) the specified virtual address range.
- *
- * - start - virtual start address
- * - end - virtual end address
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
*/
-ENTRY(v3_dma_clean_range)
+ENTRY(v3_dma_map_area)
mov pc, lr
+ENDPROC(v3_dma_unmap_area)
+ENDPROC(v3_dma_map_area)
__INITDATA
@@ -131,7 +128,7 @@ ENTRY(v3_cache_fns)
.long v3_coherent_kern_range
.long v3_coherent_user_range
.long v3_flush_kern_dcache_area
- .long v3_dma_inv_range
- .long v3_dma_clean_range
+ .long v3_dma_map_area
+ .long v3_dma_unmap_area
.long v3_dma_flush_range
.size v3_cache_fns, . - v3_cache_fns
diff --git a/arch/arm/mm/cache-v4.S b/arch/arm/mm/cache-v4.S
index 5c7da3e372e9..4810f7e3e813 100644
--- a/arch/arm/mm/cache-v4.S
+++ b/arch/arm/mm/cache-v4.S
@@ -94,20 +94,6 @@ ENTRY(v4_flush_kern_dcache_area)
/* FALLTHROUGH */
/*
- * dma_inv_range(start, end)
- *
- * Invalidate (discard) the specified virtual address range.
- * May not write back any entries. If 'start' or 'end'
- * are not cache line aligned, those lines must be written
- * back.
- *
- * - start - virtual start address
- * - end - virtual end address
- */
-ENTRY(v4_dma_inv_range)
- /* FALLTHROUGH */
-
-/*
* dma_flush_range(start, end)
*
* Clean and invalidate the specified virtual address range.
@@ -120,18 +106,29 @@ ENTRY(v4_dma_flush_range)
mov r0, #0
mcr p15, 0, r0, c7, c7, 0 @ flush ID cache
#endif
+ mov pc, lr
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(v4_dma_unmap_area)
+ teq r2, #DMA_TO_DEVICE
+ bne v4_dma_flush_range
/* FALLTHROUGH */
/*
- * dma_clean_range(start, end)
- *
- * Clean (write back) the specified virtual address range.
- *
- * - start - virtual start address
- * - end - virtual end address
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
*/
-ENTRY(v4_dma_clean_range)
+ENTRY(v4_dma_map_area)
mov pc, lr
+ENDPROC(v4_dma_unmap_area)
+ENDPROC(v4_dma_map_area)
__INITDATA
@@ -143,7 +140,7 @@ ENTRY(v4_cache_fns)
.long v4_coherent_kern_range
.long v4_coherent_user_range
.long v4_flush_kern_dcache_area
- .long v4_dma_inv_range
- .long v4_dma_clean_range
+ .long v4_dma_map_area
+ .long v4_dma_unmap_area
.long v4_dma_flush_range
.size v4_cache_fns, . - v4_cache_fns
diff --git a/arch/arm/mm/cache-v4wb.S b/arch/arm/mm/cache-v4wb.S
index 3dbedf1ec0e7..df8368afa102 100644
--- a/arch/arm/mm/cache-v4wb.S
+++ b/arch/arm/mm/cache-v4wb.S
@@ -173,7 +173,7 @@ ENTRY(v4wb_coherent_user_range)
* - start - virtual start address
* - end - virtual end address
*/
-ENTRY(v4wb_dma_inv_range)
+v4wb_dma_inv_range:
tst r0, #CACHE_DLINESIZE - 1
bic r0, r0, #CACHE_DLINESIZE - 1
mcrne p15, 0, r0, c7, c10, 1 @ clean D entry
@@ -194,7 +194,7 @@ ENTRY(v4wb_dma_inv_range)
* - start - virtual start address
* - end - virtual end address
*/
-ENTRY(v4wb_dma_clean_range)
+v4wb_dma_clean_range:
bic r0, r0, #CACHE_DLINESIZE - 1
1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry
add r0, r0, #CACHE_DLINESIZE
@@ -216,6 +216,30 @@ ENTRY(v4wb_dma_clean_range)
.globl v4wb_dma_flush_range
.set v4wb_dma_flush_range, v4wb_coherent_kern_range
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(v4wb_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq v4wb_dma_clean_range
+ bcs v4wb_dma_inv_range
+ b v4wb_dma_flush_range
+ENDPROC(v4wb_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(v4wb_dma_unmap_area)
+ mov pc, lr
+ENDPROC(v4wb_dma_unmap_area)
+
__INITDATA
.type v4wb_cache_fns, #object
@@ -226,7 +250,7 @@ ENTRY(v4wb_cache_fns)
.long v4wb_coherent_kern_range
.long v4wb_coherent_user_range
.long v4wb_flush_kern_dcache_area
- .long v4wb_dma_inv_range
- .long v4wb_dma_clean_range
+ .long v4wb_dma_map_area
+ .long v4wb_dma_unmap_area
.long v4wb_dma_flush_range
.size v4wb_cache_fns, . - v4wb_cache_fns
diff --git a/arch/arm/mm/cache-v4wt.S b/arch/arm/mm/cache-v4wt.S
index b3b7410270b4..45c70312f43b 100644
--- a/arch/arm/mm/cache-v4wt.S
+++ b/arch/arm/mm/cache-v4wt.S
@@ -142,23 +142,12 @@ ENTRY(v4wt_flush_kern_dcache_area)
* - start - virtual start address
* - end - virtual end address
*/
-ENTRY(v4wt_dma_inv_range)
+v4wt_dma_inv_range:
bic r0, r0, #CACHE_DLINESIZE - 1
1: mcr p15, 0, r0, c7, c6, 1 @ invalidate D entry
add r0, r0, #CACHE_DLINESIZE
cmp r0, r1
blo 1b
- /* FALLTHROUGH */
-
-/*
- * dma_clean_range(start, end)
- *
- * Clean the specified virtual address range.
- *
- * - start - virtual start address
- * - end - virtual end address
- */
-ENTRY(v4wt_dma_clean_range)
mov pc, lr
/*
@@ -172,6 +161,29 @@ ENTRY(v4wt_dma_clean_range)
.globl v4wt_dma_flush_range
.equ v4wt_dma_flush_range, v4wt_dma_inv_range
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(v4wt_dma_unmap_area)
+ add r1, r1, r0
+ teq r2, #DMA_TO_DEVICE
+ bne v4wt_dma_inv_range
+ /* FALLTHROUGH */
+
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(v4wt_dma_map_area)
+ mov pc, lr
+ENDPROC(v4wt_dma_unmap_area)
+ENDPROC(v4wt_dma_map_area)
+
__INITDATA
.type v4wt_cache_fns, #object
@@ -182,7 +194,7 @@ ENTRY(v4wt_cache_fns)
.long v4wt_coherent_kern_range
.long v4wt_coherent_user_range
.long v4wt_flush_kern_dcache_area
- .long v4wt_dma_inv_range
- .long v4wt_dma_clean_range
+ .long v4wt_dma_map_area
+ .long v4wt_dma_unmap_area
.long v4wt_dma_flush_range
.size v4wt_cache_fns, . - v4wt_cache_fns
diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S
index 4ba0a24ce6f5..9d89c67a1cc3 100644
--- a/arch/arm/mm/cache-v6.S
+++ b/arch/arm/mm/cache-v6.S
@@ -195,7 +195,7 @@ ENTRY(v6_flush_kern_dcache_area)
* - start - virtual start address of region
* - end - virtual end address of region
*/
-ENTRY(v6_dma_inv_range)
+v6_dma_inv_range:
tst r0, #D_CACHE_LINE_SIZE - 1
bic r0, r0, #D_CACHE_LINE_SIZE - 1
#ifdef HARVARD_CACHE
@@ -228,7 +228,7 @@ ENTRY(v6_dma_inv_range)
* - start - virtual start address of region
* - end - virtual end address of region
*/
-ENTRY(v6_dma_clean_range)
+v6_dma_clean_range:
bic r0, r0, #D_CACHE_LINE_SIZE - 1
1:
#ifdef HARVARD_CACHE
@@ -263,6 +263,32 @@ ENTRY(v6_dma_flush_range)
mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(v6_dma_map_area)
+ add r1, r1, r0
+ teq r2, #DMA_FROM_DEVICE
+ beq v6_dma_inv_range
+ b v6_dma_clean_range
+ENDPROC(v6_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(v6_dma_unmap_area)
+ add r1, r1, r0
+ teq r2, #DMA_TO_DEVICE
+ bne v6_dma_inv_range
+ mov pc, lr
+ENDPROC(v6_dma_unmap_area)
+
__INITDATA
.type v6_cache_fns, #object
@@ -273,7 +299,7 @@ ENTRY(v6_cache_fns)
.long v6_coherent_kern_range
.long v6_coherent_user_range
.long v6_flush_kern_dcache_area
- .long v6_dma_inv_range
- .long v6_dma_clean_range
+ .long v6_dma_map_area
+ .long v6_dma_unmap_area
.long v6_dma_flush_range
.size v6_cache_fns, . - v6_cache_fns
diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S
index 9073db849fb4..bcd64f265870 100644
--- a/arch/arm/mm/cache-v7.S
+++ b/arch/arm/mm/cache-v7.S
@@ -216,7 +216,7 @@ ENDPROC(v7_flush_kern_dcache_area)
* - start - virtual start address of region
* - end - virtual end address of region
*/
-ENTRY(v7_dma_inv_range)
+v7_dma_inv_range:
dcache_line_size r2, r3
sub r3, r2, #1
tst r0, r3
@@ -240,7 +240,7 @@ ENDPROC(v7_dma_inv_range)
* - start - virtual start address of region
* - end - virtual end address of region
*/
-ENTRY(v7_dma_clean_range)
+v7_dma_clean_range:
dcache_line_size r2, r3
sub r3, r2, #1
bic r0, r0, r3
@@ -271,6 +271,32 @@ ENTRY(v7_dma_flush_range)
mov pc, lr
ENDPROC(v7_dma_flush_range)
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(v7_dma_map_area)
+ add r1, r1, r0
+ teq r2, #DMA_FROM_DEVICE
+ beq v7_dma_inv_range
+ b v7_dma_clean_range
+ENDPROC(v7_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(v7_dma_unmap_area)
+ add r1, r1, r0
+ teq r2, #DMA_TO_DEVICE
+ bne v7_dma_inv_range
+ mov pc, lr
+ENDPROC(v7_dma_unmap_area)
+
__INITDATA
.type v7_cache_fns, #object
@@ -281,7 +307,7 @@ ENTRY(v7_cache_fns)
.long v7_coherent_kern_range
.long v7_coherent_user_range
.long v7_flush_kern_dcache_area
- .long v7_dma_inv_range
- .long v7_dma_clean_range
+ .long v7_dma_map_area
+ .long v7_dma_unmap_area
.long v7_dma_flush_range
.size v7_cache_fns, . - v7_cache_fns
diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c
index a9e22e31eaa1..b0ee9ba3cfab 100644
--- a/arch/arm/mm/context.c
+++ b/arch/arm/mm/context.c
@@ -10,12 +10,17 @@
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/mm.h>
+#include <linux/smp.h>
+#include <linux/percpu.h>
#include <asm/mmu_context.h>
#include <asm/tlbflush.h>
static DEFINE_SPINLOCK(cpu_asid_lock);
unsigned int cpu_last_asid = ASID_FIRST_VERSION;
+#ifdef CONFIG_SMP
+DEFINE_PER_CPU(struct mm_struct *, current_mm);
+#endif
/*
* We fork()ed a process, and we need a new context for the child
@@ -26,13 +31,109 @@ unsigned int cpu_last_asid = ASID_FIRST_VERSION;
void __init_new_context(struct task_struct *tsk, struct mm_struct *mm)
{
mm->context.id = 0;
+ spin_lock_init(&mm->context.id_lock);
}
+static void flush_context(void)
+{
+ /* set the reserved ASID before flushing the TLB */
+ asm("mcr p15, 0, %0, c13, c0, 1\n" : : "r" (0));
+ isb();
+ local_flush_tlb_all();
+ if (icache_is_vivt_asid_tagged()) {
+ __flush_icache_all();
+ dsb();
+ }
+}
+
+#ifdef CONFIG_SMP
+
+static void set_mm_context(struct mm_struct *mm, unsigned int asid)
+{
+ unsigned long flags;
+
+ /*
+ * Locking needed for multi-threaded applications where the
+ * same mm->context.id could be set from different CPUs during
+ * the broadcast. This function is also called via IPI so the
+ * mm->context.id_lock has to be IRQ-safe.
+ */
+ spin_lock_irqsave(&mm->context.id_lock, flags);
+ if (likely((mm->context.id ^ cpu_last_asid) >> ASID_BITS)) {
+ /*
+ * Old version of ASID found. Set the new one and
+ * reset mm_cpumask(mm).
+ */
+ mm->context.id = asid;
+ cpumask_clear(mm_cpumask(mm));
+ }
+ spin_unlock_irqrestore(&mm->context.id_lock, flags);
+
+ /*
+ * Set the mm_cpumask(mm) bit for the current CPU.
+ */
+ cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
+}
+
+/*
+ * Reset the ASID on the current CPU. This function call is broadcast
+ * from the CPU handling the ASID rollover and holding cpu_asid_lock.
+ */
+static void reset_context(void *info)
+{
+ unsigned int asid;
+ unsigned int cpu = smp_processor_id();
+ struct mm_struct *mm = per_cpu(current_mm, cpu);
+
+ /*
+ * Check if a current_mm was set on this CPU as it might still
+ * be in the early booting stages and using the reserved ASID.
+ */
+ if (!mm)
+ return;
+
+ smp_rmb();
+ asid = cpu_last_asid + cpu + 1;
+
+ flush_context();
+ set_mm_context(mm, asid);
+
+ /* set the new ASID */
+ asm("mcr p15, 0, %0, c13, c0, 1\n" : : "r" (mm->context.id));
+ isb();
+}
+
+#else
+
+static inline void set_mm_context(struct mm_struct *mm, unsigned int asid)
+{
+ mm->context.id = asid;
+ cpumask_copy(mm_cpumask(mm), cpumask_of(smp_processor_id()));
+}
+
+#endif
+
void __new_context(struct mm_struct *mm)
{
unsigned int asid;
spin_lock(&cpu_asid_lock);
+#ifdef CONFIG_SMP
+ /*
+ * Check the ASID again, in case the change was broadcast from
+ * another CPU before we acquired the lock.
+ */
+ if (unlikely(((mm->context.id ^ cpu_last_asid) >> ASID_BITS) == 0)) {
+ cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
+ spin_unlock(&cpu_asid_lock);
+ return;
+ }
+#endif
+ /*
+ * At this point, it is guaranteed that the current mm (with
+ * an old ASID) isn't active on any other CPU since the ASIDs
+ * are changed simultaneously via IPI.
+ */
asid = ++cpu_last_asid;
if (asid == 0)
asid = cpu_last_asid = ASID_FIRST_VERSION;
@@ -42,20 +143,15 @@ void __new_context(struct mm_struct *mm)
* to start a new version and flush the TLB.
*/
if (unlikely((asid & ~ASID_MASK) == 0)) {
- asid = ++cpu_last_asid;
- /* set the reserved ASID before flushing the TLB */
- asm("mcr p15, 0, %0, c13, c0, 1 @ set reserved context ID\n"
- :
- : "r" (0));
- isb();
- flush_tlb_all();
- if (icache_is_vivt_asid_tagged()) {
- __flush_icache_all();
- dsb();
- }
+ asid = cpu_last_asid + smp_processor_id() + 1;
+ flush_context();
+#ifdef CONFIG_SMP
+ smp_wmb();
+ smp_call_function(reset_context, NULL, 1);
+#endif
+ cpu_last_asid += NR_CPUS;
}
- spin_unlock(&cpu_asid_lock);
- cpumask_copy(mm_cpumask(mm), cpumask_of(smp_processor_id()));
- mm->context.id = asid;
+ set_mm_context(mm, asid);
+ spin_unlock(&cpu_asid_lock);
}
diff --git a/arch/arm/mm/copypage-feroceon.c b/arch/arm/mm/copypage-feroceon.c
index 70997d5bee2d..5eb4fd93893d 100644
--- a/arch/arm/mm/copypage-feroceon.c
+++ b/arch/arm/mm/copypage-feroceon.c
@@ -68,12 +68,13 @@ feroceon_copy_user_page(void *kto, const void *kfrom)
}
void feroceon_copy_user_highpage(struct page *to, struct page *from,
- unsigned long vaddr)
+ unsigned long vaddr, struct vm_area_struct *vma)
{
void *kto, *kfrom;
kto = kmap_atomic(to, KM_USER0);
kfrom = kmap_atomic(from, KM_USER1);
+ flush_cache_page(vma, vaddr, page_to_pfn(from));
feroceon_copy_user_page(kto, kfrom);
kunmap_atomic(kfrom, KM_USER1);
kunmap_atomic(kto, KM_USER0);
diff --git a/arch/arm/mm/copypage-v3.c b/arch/arm/mm/copypage-v3.c
index de9c06854ad7..f72303e1d804 100644
--- a/arch/arm/mm/copypage-v3.c
+++ b/arch/arm/mm/copypage-v3.c
@@ -38,7 +38,7 @@ v3_copy_user_page(void *kto, const void *kfrom)
}
void v3_copy_user_highpage(struct page *to, struct page *from,
- unsigned long vaddr)
+ unsigned long vaddr, struct vm_area_struct *vma)
{
void *kto, *kfrom;
diff --git a/arch/arm/mm/copypage-v4mc.c b/arch/arm/mm/copypage-v4mc.c
index 7370a7142b04..598c51ad5071 100644
--- a/arch/arm/mm/copypage-v4mc.c
+++ b/arch/arm/mm/copypage-v4mc.c
@@ -69,7 +69,7 @@ mc_copy_user_page(void *from, void *to)
}
void v4_mc_copy_user_highpage(struct page *to, struct page *from,
- unsigned long vaddr)
+ unsigned long vaddr, struct vm_area_struct *vma)
{
void *kto = kmap_atomic(to, KM_USER1);
diff --git a/arch/arm/mm/copypage-v4wb.c b/arch/arm/mm/copypage-v4wb.c
index 9ab098414227..7c2eb55cd4a9 100644
--- a/arch/arm/mm/copypage-v4wb.c
+++ b/arch/arm/mm/copypage-v4wb.c
@@ -48,12 +48,13 @@ v4wb_copy_user_page(void *kto, const void *kfrom)
}
void v4wb_copy_user_highpage(struct page *to, struct page *from,
- unsigned long vaddr)
+ unsigned long vaddr, struct vm_area_struct *vma)
{
void *kto, *kfrom;
kto = kmap_atomic(to, KM_USER0);
kfrom = kmap_atomic(from, KM_USER1);
+ flush_cache_page(vma, vaddr, page_to_pfn(from));
v4wb_copy_user_page(kto, kfrom);
kunmap_atomic(kfrom, KM_USER1);
kunmap_atomic(kto, KM_USER0);
diff --git a/arch/arm/mm/copypage-v4wt.c b/arch/arm/mm/copypage-v4wt.c
index 300efafd6643..172e6a55458e 100644
--- a/arch/arm/mm/copypage-v4wt.c
+++ b/arch/arm/mm/copypage-v4wt.c
@@ -44,7 +44,7 @@ v4wt_copy_user_page(void *kto, const void *kfrom)
}
void v4wt_copy_user_highpage(struct page *to, struct page *from,
- unsigned long vaddr)
+ unsigned long vaddr, struct vm_area_struct *vma)
{
void *kto, *kfrom;
diff --git a/arch/arm/mm/copypage-v6.c b/arch/arm/mm/copypage-v6.c
index 0fa1319273de..8bca4dea6dfa 100644
--- a/arch/arm/mm/copypage-v6.c
+++ b/arch/arm/mm/copypage-v6.c
@@ -34,7 +34,7 @@ static DEFINE_SPINLOCK(v6_lock);
* attack the kernel's existing mapping of these pages.
*/
static void v6_copy_user_highpage_nonaliasing(struct page *to,
- struct page *from, unsigned long vaddr)
+ struct page *from, unsigned long vaddr, struct vm_area_struct *vma)
{
void *kto, *kfrom;
@@ -81,7 +81,7 @@ static void discard_old_kernel_data(void *kto)
* Copy the page, taking account of the cache colour.
*/
static void v6_copy_user_highpage_aliasing(struct page *to,
- struct page *from, unsigned long vaddr)
+ struct page *from, unsigned long vaddr, struct vm_area_struct *vma)
{
unsigned int offset = CACHE_COLOUR(vaddr);
unsigned long kfrom, kto;
diff --git a/arch/arm/mm/copypage-xsc3.c b/arch/arm/mm/copypage-xsc3.c
index bc4525f5ab23..747ad4140fc7 100644
--- a/arch/arm/mm/copypage-xsc3.c
+++ b/arch/arm/mm/copypage-xsc3.c
@@ -71,12 +71,13 @@ xsc3_mc_copy_user_page(void *kto, const void *kfrom)
}
void xsc3_mc_copy_user_highpage(struct page *to, struct page *from,
- unsigned long vaddr)
+ unsigned long vaddr, struct vm_area_struct *vma)
{
void *kto, *kfrom;
kto = kmap_atomic(to, KM_USER0);
kfrom = kmap_atomic(from, KM_USER1);
+ flush_cache_page(vma, vaddr, page_to_pfn(from));
xsc3_mc_copy_user_page(kto, kfrom);
kunmap_atomic(kfrom, KM_USER1);
kunmap_atomic(kto, KM_USER0);
diff --git a/arch/arm/mm/copypage-xscale.c b/arch/arm/mm/copypage-xscale.c
index 76824d3e966a..9920c0ae2096 100644
--- a/arch/arm/mm/copypage-xscale.c
+++ b/arch/arm/mm/copypage-xscale.c
@@ -91,7 +91,7 @@ mc_copy_user_page(void *from, void *to)
}
void xscale_mc_copy_user_highpage(struct page *to, struct page *from,
- unsigned long vaddr)
+ unsigned long vaddr, struct vm_area_struct *vma)
{
void *kto = kmap_atomic(to, KM_USER1);
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index 26325cb5d368..0da7eccf7749 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -29,9 +29,6 @@
#error "CONSISTENT_DMA_SIZE must be multiple of 2MiB"
#endif
-#define CONSISTENT_END (0xffe00000)
-#define CONSISTENT_BASE (CONSISTENT_END - CONSISTENT_DMA_SIZE)
-
#define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT)
#define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT)
#define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT)
@@ -404,78 +401,44 @@ EXPORT_SYMBOL(dma_free_coherent);
* platforms with CONFIG_DMABOUNCE.
* Use the driver DMA support - see dma-mapping.h (dma_sync_*)
*/
-void dma_cache_maint(const void *start, size_t size, int direction)
+void ___dma_single_cpu_to_dev(const void *kaddr, size_t size,
+ enum dma_data_direction dir)
{
- void (*inner_op)(const void *, const void *);
- void (*outer_op)(unsigned long, unsigned long);
-
- BUG_ON(!virt_addr_valid(start) || !virt_addr_valid(start + size - 1));
-
- switch (direction) {
- case DMA_FROM_DEVICE: /* invalidate only */
- inner_op = dmac_inv_range;
- outer_op = outer_inv_range;
- break;
- case DMA_TO_DEVICE: /* writeback only */
- inner_op = dmac_clean_range;
- outer_op = outer_clean_range;
- break;
- case DMA_BIDIRECTIONAL: /* writeback and invalidate */
- inner_op = dmac_flush_range;
- outer_op = outer_flush_range;
- break;
- default:
- BUG();
- }
+ unsigned long paddr;
+
+ BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1));
- inner_op(start, start + size);
- outer_op(__pa(start), __pa(start) + size);
+ dmac_map_area(kaddr, size, dir);
+
+ paddr = __pa(kaddr);
+ if (dir == DMA_FROM_DEVICE) {
+ outer_inv_range(paddr, paddr + size);
+ } else {
+ outer_clean_range(paddr, paddr + size);
+ }
+ /* FIXME: non-speculating: flush on bidirectional mappings? */
}
-EXPORT_SYMBOL(dma_cache_maint);
+EXPORT_SYMBOL(___dma_single_cpu_to_dev);
-static void dma_cache_maint_contiguous(struct page *page, unsigned long offset,
- size_t size, int direction)
+void ___dma_single_dev_to_cpu(const void *kaddr, size_t size,
+ enum dma_data_direction dir)
{
- void *vaddr;
- unsigned long paddr;
- void (*inner_op)(const void *, const void *);
- void (*outer_op)(unsigned long, unsigned long);
-
- switch (direction) {
- case DMA_FROM_DEVICE: /* invalidate only */
- inner_op = dmac_inv_range;
- outer_op = outer_inv_range;
- break;
- case DMA_TO_DEVICE: /* writeback only */
- inner_op = dmac_clean_range;
- outer_op = outer_clean_range;
- break;
- case DMA_BIDIRECTIONAL: /* writeback and invalidate */
- inner_op = dmac_flush_range;
- outer_op = outer_flush_range;
- break;
- default:
- BUG();
- }
+ BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1));
- if (!PageHighMem(page)) {
- vaddr = page_address(page) + offset;
- inner_op(vaddr, vaddr + size);
- } else {
- vaddr = kmap_high_get(page);
- if (vaddr) {
- vaddr += offset;
- inner_op(vaddr, vaddr + size);
- kunmap_high(page);
- }
+ /* FIXME: non-speculating: not required */
+ /* don't bother invalidating if DMA to device */
+ if (dir != DMA_TO_DEVICE) {
+ unsigned long paddr = __pa(kaddr);
+ outer_inv_range(paddr, paddr + size);
}
- paddr = page_to_phys(page) + offset;
- outer_op(paddr, paddr + size);
+ dmac_unmap_area(kaddr, size, dir);
}
+EXPORT_SYMBOL(___dma_single_dev_to_cpu);
-void dma_cache_maint_page(struct page *page, unsigned long offset,
- size_t size, int dir)
+static void dma_cache_maint_page(struct page *page, unsigned long offset,
+ size_t size, enum dma_data_direction dir,
+ void (*op)(const void *, size_t, int))
{
/*
* A single sg entry may refer to multiple physically contiguous
@@ -486,20 +449,62 @@ void dma_cache_maint_page(struct page *page, unsigned long offset,
size_t left = size;
do {
size_t len = left;
- if (PageHighMem(page) && len + offset > PAGE_SIZE) {
- if (offset >= PAGE_SIZE) {
- page += offset / PAGE_SIZE;
- offset %= PAGE_SIZE;
+ void *vaddr;
+
+ if (PageHighMem(page)) {
+ if (len + offset > PAGE_SIZE) {
+ if (offset >= PAGE_SIZE) {
+ page += offset / PAGE_SIZE;
+ offset %= PAGE_SIZE;
+ }
+ len = PAGE_SIZE - offset;
}
- len = PAGE_SIZE - offset;
+ vaddr = kmap_high_get(page);
+ if (vaddr) {
+ vaddr += offset;
+ op(vaddr, len, dir);
+ kunmap_high(page);
+ }
+ } else {
+ vaddr = page_address(page) + offset;
+ op(vaddr, len, dir);
}
- dma_cache_maint_contiguous(page, offset, len, dir);
offset = 0;
page++;
left -= len;
} while (left);
}
-EXPORT_SYMBOL(dma_cache_maint_page);
+
+void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
+ size_t size, enum dma_data_direction dir)
+{
+ unsigned long paddr;
+
+ dma_cache_maint_page(page, off, size, dir, dmac_map_area);
+
+ paddr = page_to_phys(page) + off;
+ if (dir == DMA_FROM_DEVICE) {
+ outer_inv_range(paddr, paddr + size);
+ } else {
+ outer_clean_range(paddr, paddr + size);
+ }
+ /* FIXME: non-speculating: flush on bidirectional mappings? */
+}
+EXPORT_SYMBOL(___dma_page_cpu_to_dev);
+
+void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
+ size_t size, enum dma_data_direction dir)
+{
+ unsigned long paddr = page_to_phys(page) + off;
+
+ /* FIXME: non-speculating: not required */
+ /* don't bother invalidating if DMA to device */
+ if (dir != DMA_TO_DEVICE)
+ outer_inv_range(paddr, paddr + size);
+
+ dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
+}
+EXPORT_SYMBOL(___dma_page_dev_to_cpu);
/**
* dma_map_sg - map a set of SG buffers for streaming mode DMA
@@ -573,8 +578,12 @@ void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
int i;
for_each_sg(sg, s, nents, i) {
- dmabounce_sync_for_cpu(dev, sg_dma_address(s), 0,
- sg_dma_len(s), dir);
+ if (!dmabounce_sync_for_cpu(dev, sg_dma_address(s), 0,
+ sg_dma_len(s), dir))
+ continue;
+
+ __dma_page_dev_to_cpu(sg_page(s), s->offset,
+ s->length, dir);
}
}
EXPORT_SYMBOL(dma_sync_sg_for_cpu);
@@ -597,9 +606,8 @@ void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
sg_dma_len(s), dir))
continue;
- if (!arch_is_coherent())
- dma_cache_maint_page(sg_page(s), s->offset,
- s->length, dir);
+ __dma_page_cpu_to_dev(sg_page(s), s->offset,
+ s->length, dir);
}
}
EXPORT_SYMBOL(dma_sync_sg_for_device);
diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c
index 56ee15321b00..c9b97e9836a2 100644
--- a/arch/arm/mm/fault-armv.c
+++ b/arch/arm/mm/fault-armv.c
@@ -36,28 +36,12 @@ static unsigned long shared_pte_mask = L_PTE_MT_BUFFERABLE;
* Therefore those configurations which might call adjust_pte (those
* without CONFIG_CPU_CACHE_VIPT) cannot support split page_table_lock.
*/
-static int adjust_pte(struct vm_area_struct *vma, unsigned long address)
+static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address,
+ unsigned long pfn, pte_t *ptep)
{
- pgd_t *pgd;
- pmd_t *pmd;
- pte_t *pte, entry;
+ pte_t entry = *ptep;
int ret;
- pgd = pgd_offset(vma->vm_mm, address);
- if (pgd_none(*pgd))
- goto no_pgd;
- if (pgd_bad(*pgd))
- goto bad_pgd;
-
- pmd = pmd_offset(pgd, address);
- if (pmd_none(*pmd))
- goto no_pmd;
- if (pmd_bad(*pmd))
- goto bad_pmd;
-
- pte = pte_offset_map(pmd, address);
- entry = *pte;
-
/*
* If this page is present, it's actually being shared.
*/
@@ -68,33 +52,55 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address)
* fault (ie, is old), we can safely ignore any issues.
*/
if (ret && (pte_val(entry) & L_PTE_MT_MASK) != shared_pte_mask) {
- unsigned long pfn = pte_pfn(entry);
flush_cache_page(vma, address, pfn);
outer_flush_range((pfn << PAGE_SHIFT),
(pfn << PAGE_SHIFT) + PAGE_SIZE);
pte_val(entry) &= ~L_PTE_MT_MASK;
pte_val(entry) |= shared_pte_mask;
- set_pte_at(vma->vm_mm, address, pte, entry);
+ set_pte_at(vma->vm_mm, address, ptep, entry);
flush_tlb_page(vma, address);
}
- pte_unmap(pte);
+
return ret;
+}
+
+static int adjust_pte(struct vm_area_struct *vma, unsigned long address,
+ unsigned long pfn)
+{
+ spinlock_t *ptl;
+ pgd_t *pgd;
+ pmd_t *pmd;
+ pte_t *pte;
+ int ret;
+
+ pgd = pgd_offset(vma->vm_mm, address);
+ if (pgd_none_or_clear_bad(pgd))
+ return 0;
+
+ pmd = pmd_offset(pgd, address);
+ if (pmd_none_or_clear_bad(pmd))
+ return 0;
-bad_pgd:
- pgd_ERROR(*pgd);
- pgd_clear(pgd);
-no_pgd:
- return 0;
-
-bad_pmd:
- pmd_ERROR(*pmd);
- pmd_clear(pmd);
-no_pmd:
- return 0;
+ /*
+ * This is called while another page table is mapped, so we
+ * must use the nested version. This also means we need to
+ * open-code the spin-locking.
+ */
+ ptl = pte_lockptr(vma->vm_mm, pmd);
+ pte = pte_offset_map_nested(pmd, address);
+ spin_lock(ptl);
+
+ ret = do_adjust_pte(vma, address, pfn, pte);
+
+ spin_unlock(ptl);
+ pte_unmap_nested(pte);
+
+ return ret;
}
static void
-make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigned long addr, unsigned long pfn)
+make_coherent(struct address_space *mapping, struct vm_area_struct *vma,
+ unsigned long addr, pte_t *ptep, unsigned long pfn)
{
struct mm_struct *mm = vma->vm_mm;
struct vm_area_struct *mpnt;
@@ -122,11 +128,11 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigne
if (!(mpnt->vm_flags & VM_MAYSHARE))
continue;
offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT;
- aliases += adjust_pte(mpnt, mpnt->vm_start + offset);
+ aliases += adjust_pte(mpnt, mpnt->vm_start + offset, pfn);
}
flush_dcache_mmap_unlock(mapping);
if (aliases)
- adjust_pte(vma, addr);
+ do_adjust_pte(vma, addr, pfn, ptep);
else
flush_cache_page(vma, addr, pfn);
}
@@ -144,9 +150,10 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigne
*
* Note that the pte lock will be held.
*/
-void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
+void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr,
+ pte_t *ptep)
{
- unsigned long pfn = pte_pfn(pte);
+ unsigned long pfn = pte_pfn(*ptep);
struct address_space *mapping;
struct page *page;
@@ -168,7 +175,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
#endif
if (mapping) {
if (cache_is_vivt())
- make_coherent(mapping, vma, addr, pfn);
+ make_coherent(mapping, vma, addr, ptep, pfn);
else if (vma->vm_flags & VM_EXEC)
__flush_icache_all();
}
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index 10e06801afb3..9d40c341e07e 100644
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -18,6 +18,7 @@
#include <linux/page-flags.h>
#include <linux/sched.h>
#include <linux/highmem.h>
+#include <linux/perf_event.h>
#include <asm/system.h>
#include <asm/pgtable.h>
@@ -302,6 +303,12 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
fault = __do_page_fault(mm, addr, fsr, tsk);
up_read(&mm->mmap_sem);
+ perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, addr);
+ if (fault & VM_FAULT_MAJOR)
+ perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, regs, addr);
+ else if (fault & VM_FAULT_MINOR)
+ perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, regs, addr);
+
/*
* Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR
*/
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index 6f3a4b7a3b82..e34f095e2090 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -13,6 +13,7 @@
#include <asm/cacheflush.h>
#include <asm/cachetype.h>
+#include <asm/smp_plat.h>
#include <asm/system.h>
#include <asm/tlbflush.h>
@@ -87,13 +88,26 @@ void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsig
if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged())
__flush_icache_all();
}
+#else
+#define flush_pfn_alias(pfn,vaddr) do { } while (0)
+#endif
+#ifdef CONFIG_SMP
+static void flush_ptrace_access_other(void *args)
+{
+ __flush_icache_all();
+}
+#endif
+
+static
void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
- unsigned long uaddr, void *kaddr,
- unsigned long len, int write)
+ unsigned long uaddr, void *kaddr, unsigned long len)
{
if (cache_is_vivt()) {
- vivt_flush_ptrace_access(vma, page, uaddr, kaddr, len, write);
+ if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) {
+ unsigned long addr = (unsigned long)kaddr;
+ __cpuc_coherent_kern_range(addr, addr + len);
+ }
return;
}
@@ -104,16 +118,37 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
}
/* VIPT non-aliasing cache */
- if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm)) &&
- vma->vm_flags & VM_EXEC) {
+ if (vma->vm_flags & VM_EXEC) {
unsigned long addr = (unsigned long)kaddr;
- /* only flushing the kernel mapping on non-aliasing VIPT */
__cpuc_coherent_kern_range(addr, addr + len);
+#ifdef CONFIG_SMP
+ if (cache_ops_need_broadcast())
+ smp_call_function(flush_ptrace_access_other,
+ NULL, 1);
+#endif
}
}
-#else
-#define flush_pfn_alias(pfn,vaddr) do { } while (0)
+
+/*
+ * Copy user data from/to a page which is mapped into a different
+ * processes address space. Really, we want to allow our "user
+ * space" model to handle this.
+ *
+ * Note that this code needs to run on the current CPU.
+ */
+void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
+ unsigned long uaddr, void *dst, const void *src,
+ unsigned long len)
+{
+#ifdef CONFIG_SMP
+ preempt_disable();
#endif
+ memcpy(dst, src, len);
+ flush_ptrace_access(vma, page, uaddr, dst, len);
+#ifdef CONFIG_SMP
+ preempt_enable();
+#endif
+}
void __flush_dcache_page(struct address_space *mapping, struct page *page)
{
diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index a04ffbbbe253..7829cb5425f5 100644
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -23,6 +23,7 @@
#include <asm/setup.h>
#include <asm/sizes.h>
#include <asm/tlb.h>
+#include <asm/fixmap.h>
#include <asm/mach/arch.h>
#include <asm/mach/map.h>
@@ -32,19 +33,21 @@
static unsigned long phys_initrd_start __initdata = 0;
static unsigned long phys_initrd_size __initdata = 0;
-static void __init early_initrd(char **p)
+static int __init early_initrd(char *p)
{
unsigned long start, size;
+ char *endp;
- start = memparse(*p, p);
- if (**p == ',') {
- size = memparse((*p) + 1, p);
+ start = memparse(p, &endp);
+ if (*endp == ',') {
+ size = memparse(endp + 1, NULL);
phys_initrd_start = start;
phys_initrd_size = size;
}
+ return 0;
}
-__early_param("initrd=", early_initrd);
+early_param("initrd", early_initrd);
static int __init parse_tag_initrd(const struct tag *tag)
{
@@ -560,7 +563,7 @@ static void __init free_unused_memmap_node(int node, struct meminfo *mi)
*/
void __init mem_init(void)
{
- unsigned int codesize, datasize, initsize;
+ unsigned long reserved_pages, free_pages;
int i, node;
#ifndef CONFIG_DISCONTIGMEM
@@ -596,6 +599,33 @@ void __init mem_init(void)
totalram_pages += totalhigh_pages;
#endif
+ reserved_pages = free_pages = 0;
+
+ for_each_online_node(node) {
+ pg_data_t *n = NODE_DATA(node);
+ struct page *map = pgdat_page_nr(n, 0) - n->node_start_pfn;
+
+ for_each_nodebank(i, &meminfo, node) {
+ struct membank *bank = &meminfo.bank[i];
+ unsigned int pfn1, pfn2;
+ struct page *page, *end;
+
+ pfn1 = bank_pfn_start(bank);
+ pfn2 = bank_pfn_end(bank);
+
+ page = map + pfn1;
+ end = map + pfn2;
+
+ do {
+ if (PageReserved(page))
+ reserved_pages++;
+ else if (!page_count(page))
+ free_pages++;
+ page++;
+ } while (page < end);
+ }
+ }
+
/*
* Since our memory may not be contiguous, calculate the
* real number of pages we have in this system
@@ -608,16 +638,71 @@ void __init mem_init(void)
}
printk(" = %luMB total\n", num_physpages >> (20 - PAGE_SHIFT));
- codesize = _etext - _text;
- datasize = _end - _data;
- initsize = __init_end - __init_begin;
-
- printk(KERN_NOTICE "Memory: %luKB available (%dK code, "
- "%dK data, %dK init, %luK highmem)\n",
- nr_free_pages() << (PAGE_SHIFT-10), codesize >> 10,
- datasize >> 10, initsize >> 10,
+ printk(KERN_NOTICE "Memory: %luk/%luk available, %luk reserved, %luK highmem\n",
+ nr_free_pages() << (PAGE_SHIFT-10),
+ free_pages << (PAGE_SHIFT-10),
+ reserved_pages << (PAGE_SHIFT-10),
totalhigh_pages << (PAGE_SHIFT-10));
+#define MLK(b, t) b, t, ((t) - (b)) >> 10
+#define MLM(b, t) b, t, ((t) - (b)) >> 20
+#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)
+
+ printk(KERN_NOTICE "Virtual kernel memory layout:\n"
+ " vector : 0x%08lx - 0x%08lx (%4ld kB)\n"
+ " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
+#ifdef CONFIG_MMU
+ " DMA : 0x%08lx - 0x%08lx (%4ld MB)\n"
+#endif
+ " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n"
+ " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n"
+#ifdef CONFIG_HIGHMEM
+ " pkmap : 0x%08lx - 0x%08lx (%4ld MB)\n"
+#endif
+ " modules : 0x%08lx - 0x%08lx (%4ld MB)\n"
+ " .init : 0x%p" " - 0x%p" " (%4d kB)\n"
+ " .text : 0x%p" " - 0x%p" " (%4d kB)\n"
+ " .data : 0x%p" " - 0x%p" " (%4d kB)\n",
+
+ MLK(UL(CONFIG_VECTORS_BASE), UL(CONFIG_VECTORS_BASE) +
+ (PAGE_SIZE)),
+ MLK(FIXADDR_START, FIXADDR_TOP),
+#ifdef CONFIG_MMU
+ MLM(CONSISTENT_BASE, CONSISTENT_END),
+#endif
+ MLM(VMALLOC_START, VMALLOC_END),
+ MLM(PAGE_OFFSET, (unsigned long)high_memory),
+#ifdef CONFIG_HIGHMEM
+ MLM(PKMAP_BASE, (PKMAP_BASE) + (LAST_PKMAP) *
+ (PAGE_SIZE)),
+#endif
+ MLM(MODULES_VADDR, MODULES_END),
+
+ MLK_ROUNDUP(__init_begin, __init_end),
+ MLK_ROUNDUP(_text, _etext),
+ MLK_ROUNDUP(_data, _edata));
+
+#undef MLK
+#undef MLM
+#undef MLK_ROUNDUP
+
+ /*
+ * Check boundaries twice: Some fundamental inconsistencies can
+ * be detected at build time already.
+ */
+#ifdef CONFIG_MMU
+ BUILD_BUG_ON(VMALLOC_END > CONSISTENT_BASE);
+ BUG_ON(VMALLOC_END > CONSISTENT_BASE);
+
+ BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR);
+ BUG_ON(TASK_SIZE > MODULES_VADDR);
+#endif
+
+#ifdef CONFIG_HIGHMEM
+ BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET);
+ BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET);
+#endif
+
if (PAGE_SIZE >= 16384 && num_physpages <= 128) {
extern int sysctl_overcommit_memory;
/*
diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c
index 0ab75c60f7cf..28c8b950ef04 100644
--- a/arch/arm/mm/ioremap.c
+++ b/arch/arm/mm/ioremap.c
@@ -139,8 +139,8 @@ void __check_kvm_seq(struct mm_struct *mm)
* which requires the new ioremap'd region to be referenced, the CPU will
* reference the _old_ region.
*
- * Note that get_vm_area() allocates a guard 4K page, so we need to mask
- * the size back to 1MB aligned or we will overflow in the loop below.
+ * Note that get_vm_area_caller() allocates a guard 4K page, so we need to
+ * mask the size back to 1MB aligned or we will overflow in the loop below.
*/
static void unmap_area_sections(unsigned long virt, unsigned long size)
{
@@ -254,22 +254,8 @@ remap_area_supersections(unsigned long virt, unsigned long pfn,
}
#endif
-
-/*
- * Remap an arbitrary physical address space into the kernel virtual
- * address space. Needed when the kernel wants to access high addresses
- * directly.
- *
- * NOTE! We need to allow non-page-aligned mappings too: we will obviously
- * have to convert them into an offset in a page-aligned mapping, but the
- * caller shouldn't need to know that small detail.
- *
- * 'flags' are the extra L_PTE_ flags that you want to specify for this
- * mapping. See <asm/pgtable.h> for more information.
- */
-void __iomem *
-__arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size,
- unsigned int mtype)
+void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn,
+ unsigned long offset, size_t size, unsigned int mtype, void *caller)
{
const struct mem_type *type;
int err;
@@ -291,7 +277,7 @@ __arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size,
*/
size = PAGE_ALIGN(offset + size);
- area = get_vm_area(size, VM_IOREMAP);
+ area = get_vm_area_caller(size, VM_IOREMAP, caller);
if (!area)
return NULL;
addr = (unsigned long)area->addr;
@@ -318,10 +304,9 @@ __arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size,
flush_cache_vmap(addr, addr + size);
return (void __iomem *) (offset + addr);
}
-EXPORT_SYMBOL(__arm_ioremap_pfn);
-void __iomem *
-__arm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype)
+void __iomem *__arm_ioremap_caller(unsigned long phys_addr, size_t size,
+ unsigned int mtype, void *caller)
{
unsigned long last_addr;
unsigned long offset = phys_addr & ~PAGE_MASK;
@@ -334,7 +319,33 @@ __arm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype)
if (!size || last_addr < phys_addr)
return NULL;
- return __arm_ioremap_pfn(pfn, offset, size, mtype);
+ return __arm_ioremap_pfn_caller(pfn, offset, size, mtype,
+ caller);
+}
+
+/*
+ * Remap an arbitrary physical address space into the kernel virtual
+ * address space. Needed when the kernel wants to access high addresses
+ * directly.
+ *
+ * NOTE! We need to allow non-page-aligned mappings too: we will obviously
+ * have to convert them into an offset in a page-aligned mapping, but the
+ * caller shouldn't need to know that small detail.
+ */
+void __iomem *
+__arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size,
+ unsigned int mtype)
+{
+ return __arm_ioremap_pfn_caller(pfn, offset, size, mtype,
+ __builtin_return_address(0));
+}
+EXPORT_SYMBOL(__arm_ioremap_pfn);
+
+void __iomem *
+__arm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype)
+{
+ return __arm_ioremap_caller(phys_addr, size, mtype,
+ __builtin_return_address(0));
}
EXPORT_SYMBOL(__arm_ioremap);
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index 761ffede6a23..9d4da6ac28eb 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -100,18 +100,17 @@ static struct cachepolicy cache_policies[] __initdata = {
* writebuffer to be turned off. (Note: the write
* buffer should not be on and the cache off).
*/
-static void __init early_cachepolicy(char **p)
+static int __init early_cachepolicy(char *p)
{
int i;
for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
int len = strlen(cache_policies[i].policy);
- if (memcmp(*p, cache_policies[i].policy, len) == 0) {
+ if (memcmp(p, cache_policies[i].policy, len) == 0) {
cachepolicy = i;
cr_alignment &= ~cache_policies[i].cr_mask;
cr_no_alignment &= ~cache_policies[i].cr_mask;
- *p += len;
break;
}
}
@@ -130,36 +129,37 @@ static void __init early_cachepolicy(char **p)
}
flush_cache_all();
set_cr(cr_alignment);
+ return 0;
}
-__early_param("cachepolicy=", early_cachepolicy);
+early_param("cachepolicy", early_cachepolicy);
-static void __init early_nocache(char **__unused)
+static int __init early_nocache(char *__unused)
{
char *p = "buffered";
printk(KERN_WARNING "nocache is deprecated; use cachepolicy=%s\n", p);
- early_cachepolicy(&p);
+ early_cachepolicy(p);
+ return 0;
}
-__early_param("nocache", early_nocache);
+early_param("nocache", early_nocache);
-static void __init early_nowrite(char **__unused)
+static int __init early_nowrite(char *__unused)
{
char *p = "uncached";
printk(KERN_WARNING "nowb is deprecated; use cachepolicy=%s\n", p);
- early_cachepolicy(&p);
+ early_cachepolicy(p);
+ return 0;
}
-__early_param("nowb", early_nowrite);
+early_param("nowb", early_nowrite);
-static void __init early_ecc(char **p)
+static int __init early_ecc(char *p)
{
- if (memcmp(*p, "on", 2) == 0) {
+ if (memcmp(p, "on", 2) == 0)
ecc_mask = PMD_PROTECTION;
- *p += 2;
- } else if (memcmp(*p, "off", 3) == 0) {
+ else if (memcmp(p, "off", 3) == 0)
ecc_mask = 0;
- *p += 3;
- }
+ return 0;
}
-__early_param("ecc=", early_ecc);
+early_param("ecc", early_ecc);
static int __init noalign_setup(char *__unused)
{
@@ -670,9 +670,9 @@ static unsigned long __initdata vmalloc_reserve = SZ_128M;
* bytes. This can be used to increase (or decrease) the vmalloc
* area - the default is 128m.
*/
-static void __init early_vmalloc(char **arg)
+static int __init early_vmalloc(char *arg)
{
- vmalloc_reserve = memparse(*arg, arg);
+ vmalloc_reserve = memparse(arg, NULL);
if (vmalloc_reserve < SZ_16M) {
vmalloc_reserve = SZ_16M;
@@ -687,8 +687,9 @@ static void __init early_vmalloc(char **arg)
"vmalloc area is too big, limiting to %luMB\n",
vmalloc_reserve >> 20);
}
+ return 0;
}
-__early_param("vmalloc=", early_vmalloc);
+early_param("vmalloc", early_vmalloc);
#define VMALLOC_MIN (void *)(VMALLOC_END - vmalloc_reserve)
diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c
index 374a8311bc84..9bfeb6b9509a 100644
--- a/arch/arm/mm/nommu.c
+++ b/arch/arm/mm/nommu.c
@@ -74,6 +74,12 @@ void __iomem *__arm_ioremap_pfn(unsigned long pfn, unsigned long offset,
}
EXPORT_SYMBOL(__arm_ioremap_pfn);
+void __iomem *__arm_ioremap_pfn_caller(unsigned long pfn, unsigned long offset,
+ size_t size, unsigned int mtype, void *caller)
+{
+ return __arm_ioremap_pfn(pfn, offset, size, mtype);
+}
+
void __iomem *__arm_ioremap(unsigned long phys_addr, size_t size,
unsigned int mtype)
{
@@ -81,6 +87,12 @@ void __iomem *__arm_ioremap(unsigned long phys_addr, size_t size,
}
EXPORT_SYMBOL(__arm_ioremap);
+void __iomem *__arm_ioremap(unsigned long phys_addr, size_t size,
+ unsigned int mtype, void *caller)
+{
+ return __arm_ioremap(phys_addr, size, mtype);
+}
+
void __iounmap(volatile void __iomem *addr)
{
}
diff --git a/arch/arm/mm/proc-arm1020.S b/arch/arm/mm/proc-arm1020.S
index 8012e24282b2..72507c630ceb 100644
--- a/arch/arm/mm/proc-arm1020.S
+++ b/arch/arm/mm/proc-arm1020.S
@@ -265,7 +265,7 @@ ENTRY(arm1020_flush_kern_dcache_area)
*
* (same as v4wb)
*/
-ENTRY(arm1020_dma_inv_range)
+arm1020_dma_inv_range:
mov ip, #0
#ifndef CONFIG_CPU_DCACHE_DISABLE
tst r0, #CACHE_DLINESIZE - 1
@@ -295,7 +295,7 @@ ENTRY(arm1020_dma_inv_range)
*
* (same as v4wb)
*/
-ENTRY(arm1020_dma_clean_range)
+arm1020_dma_clean_range:
mov ip, #0
#ifndef CONFIG_CPU_DCACHE_DISABLE
bic r0, r0, #CACHE_DLINESIZE - 1
@@ -330,6 +330,30 @@ ENTRY(arm1020_dma_flush_range)
mcr p15, 0, ip, c7, c10, 4 @ drain WB
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm1020_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq arm1020_dma_clean_range
+ bcs arm1020_dma_inv_range
+ b arm1020_dma_flush_range
+ENDPROC(arm1020_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm1020_dma_unmap_area)
+ mov pc, lr
+ENDPROC(arm1020_dma_unmap_area)
+
ENTRY(arm1020_cache_fns)
.long arm1020_flush_kern_cache_all
.long arm1020_flush_user_cache_all
@@ -337,8 +361,8 @@ ENTRY(arm1020_cache_fns)
.long arm1020_coherent_kern_range
.long arm1020_coherent_user_range
.long arm1020_flush_kern_dcache_area
- .long arm1020_dma_inv_range
- .long arm1020_dma_clean_range
+ .long arm1020_dma_map_area
+ .long arm1020_dma_unmap_area
.long arm1020_dma_flush_range
.align 5
diff --git a/arch/arm/mm/proc-arm1020e.S b/arch/arm/mm/proc-arm1020e.S
index 41fe25d234f5..d27829805609 100644
--- a/arch/arm/mm/proc-arm1020e.S
+++ b/arch/arm/mm/proc-arm1020e.S
@@ -258,7 +258,7 @@ ENTRY(arm1020e_flush_kern_dcache_area)
*
* (same as v4wb)
*/
-ENTRY(arm1020e_dma_inv_range)
+arm1020e_dma_inv_range:
mov ip, #0
#ifndef CONFIG_CPU_DCACHE_DISABLE
tst r0, #CACHE_DLINESIZE - 1
@@ -284,7 +284,7 @@ ENTRY(arm1020e_dma_inv_range)
*
* (same as v4wb)
*/
-ENTRY(arm1020e_dma_clean_range)
+arm1020e_dma_clean_range:
mov ip, #0
#ifndef CONFIG_CPU_DCACHE_DISABLE
bic r0, r0, #CACHE_DLINESIZE - 1
@@ -316,6 +316,30 @@ ENTRY(arm1020e_dma_flush_range)
mcr p15, 0, ip, c7, c10, 4 @ drain WB
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm1020e_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq arm1020e_dma_clean_range
+ bcs arm1020e_dma_inv_range
+ b arm1020e_dma_flush_range
+ENDPROC(arm1020e_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm1020e_dma_unmap_area)
+ mov pc, lr
+ENDPROC(arm1020e_dma_unmap_area)
+
ENTRY(arm1020e_cache_fns)
.long arm1020e_flush_kern_cache_all
.long arm1020e_flush_user_cache_all
@@ -323,8 +347,8 @@ ENTRY(arm1020e_cache_fns)
.long arm1020e_coherent_kern_range
.long arm1020e_coherent_user_range
.long arm1020e_flush_kern_dcache_area
- .long arm1020e_dma_inv_range
- .long arm1020e_dma_clean_range
+ .long arm1020e_dma_map_area
+ .long arm1020e_dma_unmap_area
.long arm1020e_dma_flush_range
.align 5
diff --git a/arch/arm/mm/proc-arm1022.S b/arch/arm/mm/proc-arm1022.S
index 20a5b1b31a70..ce13e4a827de 100644
--- a/arch/arm/mm/proc-arm1022.S
+++ b/arch/arm/mm/proc-arm1022.S
@@ -247,7 +247,7 @@ ENTRY(arm1022_flush_kern_dcache_area)
*
* (same as v4wb)
*/
-ENTRY(arm1022_dma_inv_range)
+arm1022_dma_inv_range:
mov ip, #0
#ifndef CONFIG_CPU_DCACHE_DISABLE
tst r0, #CACHE_DLINESIZE - 1
@@ -273,7 +273,7 @@ ENTRY(arm1022_dma_inv_range)
*
* (same as v4wb)
*/
-ENTRY(arm1022_dma_clean_range)
+arm1022_dma_clean_range:
mov ip, #0
#ifndef CONFIG_CPU_DCACHE_DISABLE
bic r0, r0, #CACHE_DLINESIZE - 1
@@ -305,6 +305,30 @@ ENTRY(arm1022_dma_flush_range)
mcr p15, 0, ip, c7, c10, 4 @ drain WB
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm1022_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq arm1022_dma_clean_range
+ bcs arm1022_dma_inv_range
+ b arm1022_dma_flush_range
+ENDPROC(arm1022_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm1022_dma_unmap_area)
+ mov pc, lr
+ENDPROC(arm1022_dma_unmap_area)
+
ENTRY(arm1022_cache_fns)
.long arm1022_flush_kern_cache_all
.long arm1022_flush_user_cache_all
@@ -312,8 +336,8 @@ ENTRY(arm1022_cache_fns)
.long arm1022_coherent_kern_range
.long arm1022_coherent_user_range
.long arm1022_flush_kern_dcache_area
- .long arm1022_dma_inv_range
- .long arm1022_dma_clean_range
+ .long arm1022_dma_map_area
+ .long arm1022_dma_unmap_area
.long arm1022_dma_flush_range
.align 5
diff --git a/arch/arm/mm/proc-arm1026.S b/arch/arm/mm/proc-arm1026.S
index 96aedb10fcc4..636672a29c6d 100644
--- a/arch/arm/mm/proc-arm1026.S
+++ b/arch/arm/mm/proc-arm1026.S
@@ -241,7 +241,7 @@ ENTRY(arm1026_flush_kern_dcache_area)
*
* (same as v4wb)
*/
-ENTRY(arm1026_dma_inv_range)
+arm1026_dma_inv_range:
mov ip, #0
#ifndef CONFIG_CPU_DCACHE_DISABLE
tst r0, #CACHE_DLINESIZE - 1
@@ -267,7 +267,7 @@ ENTRY(arm1026_dma_inv_range)
*
* (same as v4wb)
*/
-ENTRY(arm1026_dma_clean_range)
+arm1026_dma_clean_range:
mov ip, #0
#ifndef CONFIG_CPU_DCACHE_DISABLE
bic r0, r0, #CACHE_DLINESIZE - 1
@@ -299,6 +299,30 @@ ENTRY(arm1026_dma_flush_range)
mcr p15, 0, ip, c7, c10, 4 @ drain WB
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm1026_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq arm1026_dma_clean_range
+ bcs arm1026_dma_inv_range
+ b arm1026_dma_flush_range
+ENDPROC(arm1026_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm1026_dma_unmap_area)
+ mov pc, lr
+ENDPROC(arm1026_dma_unmap_area)
+
ENTRY(arm1026_cache_fns)
.long arm1026_flush_kern_cache_all
.long arm1026_flush_user_cache_all
@@ -306,8 +330,8 @@ ENTRY(arm1026_cache_fns)
.long arm1026_coherent_kern_range
.long arm1026_coherent_user_range
.long arm1026_flush_kern_dcache_area
- .long arm1026_dma_inv_range
- .long arm1026_dma_clean_range
+ .long arm1026_dma_map_area
+ .long arm1026_dma_unmap_area
.long arm1026_dma_flush_range
.align 5
diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S
index 471669e2d7cb..8be81992645d 100644
--- a/arch/arm/mm/proc-arm920.S
+++ b/arch/arm/mm/proc-arm920.S
@@ -239,7 +239,7 @@ ENTRY(arm920_flush_kern_dcache_area)
*
* (same as v4wb)
*/
-ENTRY(arm920_dma_inv_range)
+arm920_dma_inv_range:
tst r0, #CACHE_DLINESIZE - 1
bic r0, r0, #CACHE_DLINESIZE - 1
mcrne p15, 0, r0, c7, c10, 1 @ clean D entry
@@ -262,7 +262,7 @@ ENTRY(arm920_dma_inv_range)
*
* (same as v4wb)
*/
-ENTRY(arm920_dma_clean_range)
+arm920_dma_clean_range:
bic r0, r0, #CACHE_DLINESIZE - 1
1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry
add r0, r0, #CACHE_DLINESIZE
@@ -288,6 +288,30 @@ ENTRY(arm920_dma_flush_range)
mcr p15, 0, r0, c7, c10, 4 @ drain WB
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm920_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq arm920_dma_clean_range
+ bcs arm920_dma_inv_range
+ b arm920_dma_flush_range
+ENDPROC(arm920_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm920_dma_unmap_area)
+ mov pc, lr
+ENDPROC(arm920_dma_unmap_area)
+
ENTRY(arm920_cache_fns)
.long arm920_flush_kern_cache_all
.long arm920_flush_user_cache_all
@@ -295,8 +319,8 @@ ENTRY(arm920_cache_fns)
.long arm920_coherent_kern_range
.long arm920_coherent_user_range
.long arm920_flush_kern_dcache_area
- .long arm920_dma_inv_range
- .long arm920_dma_clean_range
+ .long arm920_dma_map_area
+ .long arm920_dma_unmap_area
.long arm920_dma_flush_range
#endif
diff --git a/arch/arm/mm/proc-arm922.S b/arch/arm/mm/proc-arm922.S
index ee111b00fa41..c0ff8e4b1074 100644
--- a/arch/arm/mm/proc-arm922.S
+++ b/arch/arm/mm/proc-arm922.S
@@ -241,7 +241,7 @@ ENTRY(arm922_flush_kern_dcache_area)
*
* (same as v4wb)
*/
-ENTRY(arm922_dma_inv_range)
+arm922_dma_inv_range:
tst r0, #CACHE_DLINESIZE - 1
bic r0, r0, #CACHE_DLINESIZE - 1
mcrne p15, 0, r0, c7, c10, 1 @ clean D entry
@@ -264,7 +264,7 @@ ENTRY(arm922_dma_inv_range)
*
* (same as v4wb)
*/
-ENTRY(arm922_dma_clean_range)
+arm922_dma_clean_range:
bic r0, r0, #CACHE_DLINESIZE - 1
1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry
add r0, r0, #CACHE_DLINESIZE
@@ -290,6 +290,30 @@ ENTRY(arm922_dma_flush_range)
mcr p15, 0, r0, c7, c10, 4 @ drain WB
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm922_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq arm922_dma_clean_range
+ bcs arm922_dma_inv_range
+ b arm922_dma_flush_range
+ENDPROC(arm922_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm922_dma_unmap_area)
+ mov pc, lr
+ENDPROC(arm922_dma_unmap_area)
+
ENTRY(arm922_cache_fns)
.long arm922_flush_kern_cache_all
.long arm922_flush_user_cache_all
@@ -297,8 +321,8 @@ ENTRY(arm922_cache_fns)
.long arm922_coherent_kern_range
.long arm922_coherent_user_range
.long arm922_flush_kern_dcache_area
- .long arm922_dma_inv_range
- .long arm922_dma_clean_range
+ .long arm922_dma_map_area
+ .long arm922_dma_unmap_area
.long arm922_dma_flush_range
#endif
diff --git a/arch/arm/mm/proc-arm925.S b/arch/arm/mm/proc-arm925.S
index 8deb5bde58e4..3c6cffe400f6 100644
--- a/arch/arm/mm/proc-arm925.S
+++ b/arch/arm/mm/proc-arm925.S
@@ -283,7 +283,7 @@ ENTRY(arm925_flush_kern_dcache_area)
*
* (same as v4wb)
*/
-ENTRY(arm925_dma_inv_range)
+arm925_dma_inv_range:
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
tst r0, #CACHE_DLINESIZE - 1
mcrne p15, 0, r0, c7, c10, 1 @ clean D entry
@@ -308,7 +308,7 @@ ENTRY(arm925_dma_inv_range)
*
* (same as v4wb)
*/
-ENTRY(arm925_dma_clean_range)
+arm925_dma_clean_range:
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
bic r0, r0, #CACHE_DLINESIZE - 1
1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry
@@ -341,6 +341,30 @@ ENTRY(arm925_dma_flush_range)
mcr p15, 0, r0, c7, c10, 4 @ drain WB
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm925_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq arm925_dma_clean_range
+ bcs arm925_dma_inv_range
+ b arm925_dma_flush_range
+ENDPROC(arm925_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm925_dma_unmap_area)
+ mov pc, lr
+ENDPROC(arm925_dma_unmap_area)
+
ENTRY(arm925_cache_fns)
.long arm925_flush_kern_cache_all
.long arm925_flush_user_cache_all
@@ -348,8 +372,8 @@ ENTRY(arm925_cache_fns)
.long arm925_coherent_kern_range
.long arm925_coherent_user_range
.long arm925_flush_kern_dcache_area
- .long arm925_dma_inv_range
- .long arm925_dma_clean_range
+ .long arm925_dma_map_area
+ .long arm925_dma_unmap_area
.long arm925_dma_flush_range
ENTRY(cpu_arm925_dcache_clean_area)
diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S
index 64db6e275a44..75b707c9cce1 100644
--- a/arch/arm/mm/proc-arm926.S
+++ b/arch/arm/mm/proc-arm926.S
@@ -246,7 +246,7 @@ ENTRY(arm926_flush_kern_dcache_area)
*
* (same as v4wb)
*/
-ENTRY(arm926_dma_inv_range)
+arm926_dma_inv_range:
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
tst r0, #CACHE_DLINESIZE - 1
mcrne p15, 0, r0, c7, c10, 1 @ clean D entry
@@ -271,7 +271,7 @@ ENTRY(arm926_dma_inv_range)
*
* (same as v4wb)
*/
-ENTRY(arm926_dma_clean_range)
+arm926_dma_clean_range:
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
bic r0, r0, #CACHE_DLINESIZE - 1
1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry
@@ -304,6 +304,30 @@ ENTRY(arm926_dma_flush_range)
mcr p15, 0, r0, c7, c10, 4 @ drain WB
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm926_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq arm926_dma_clean_range
+ bcs arm926_dma_inv_range
+ b arm926_dma_flush_range
+ENDPROC(arm926_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm926_dma_unmap_area)
+ mov pc, lr
+ENDPROC(arm926_dma_unmap_area)
+
ENTRY(arm926_cache_fns)
.long arm926_flush_kern_cache_all
.long arm926_flush_user_cache_all
@@ -311,8 +335,8 @@ ENTRY(arm926_cache_fns)
.long arm926_coherent_kern_range
.long arm926_coherent_user_range
.long arm926_flush_kern_dcache_area
- .long arm926_dma_inv_range
- .long arm926_dma_clean_range
+ .long arm926_dma_map_area
+ .long arm926_dma_unmap_area
.long arm926_dma_flush_range
ENTRY(cpu_arm926_dcache_clean_area)
diff --git a/arch/arm/mm/proc-arm940.S b/arch/arm/mm/proc-arm940.S
index 8196b9f401fb..1af1657819eb 100644
--- a/arch/arm/mm/proc-arm940.S
+++ b/arch/arm/mm/proc-arm940.S
@@ -171,7 +171,7 @@ ENTRY(arm940_flush_kern_dcache_area)
* - start - virtual start address
* - end - virtual end address
*/
-ENTRY(arm940_dma_inv_range)
+arm940_dma_inv_range:
mov ip, #0
mov r1, #(CACHE_DSEGMENTS - 1) << 4 @ 4 segments
1: orr r3, r1, #(CACHE_DENTRIES - 1) << 26 @ 64 entries
@@ -192,7 +192,7 @@ ENTRY(arm940_dma_inv_range)
* - start - virtual start address
* - end - virtual end address
*/
-ENTRY(arm940_dma_clean_range)
+arm940_dma_clean_range:
ENTRY(cpu_arm940_dcache_clean_area)
mov ip, #0
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
@@ -233,6 +233,30 @@ ENTRY(arm940_dma_flush_range)
mcr p15, 0, ip, c7, c10, 4 @ drain WB
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm940_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq arm940_dma_clean_range
+ bcs arm940_dma_inv_range
+ b arm940_dma_flush_range
+ENDPROC(arm940_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm940_dma_unmap_area)
+ mov pc, lr
+ENDPROC(arm940_dma_unmap_area)
+
ENTRY(arm940_cache_fns)
.long arm940_flush_kern_cache_all
.long arm940_flush_user_cache_all
@@ -240,8 +264,8 @@ ENTRY(arm940_cache_fns)
.long arm940_coherent_kern_range
.long arm940_coherent_user_range
.long arm940_flush_kern_dcache_area
- .long arm940_dma_inv_range
- .long arm940_dma_clean_range
+ .long arm940_dma_map_area
+ .long arm940_dma_unmap_area
.long arm940_dma_flush_range
__INIT
diff --git a/arch/arm/mm/proc-arm946.S b/arch/arm/mm/proc-arm946.S
index 9a951239c86c..1664b6aaff79 100644
--- a/arch/arm/mm/proc-arm946.S
+++ b/arch/arm/mm/proc-arm946.S
@@ -215,7 +215,7 @@ ENTRY(arm946_flush_kern_dcache_area)
* - end - virtual end address
* (same as arm926)
*/
-ENTRY(arm946_dma_inv_range)
+arm946_dma_inv_range:
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
tst r0, #CACHE_DLINESIZE - 1
mcrne p15, 0, r0, c7, c10, 1 @ clean D entry
@@ -240,7 +240,7 @@ ENTRY(arm946_dma_inv_range)
*
* (same as arm926)
*/
-ENTRY(arm946_dma_clean_range)
+arm946_dma_clean_range:
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
bic r0, r0, #CACHE_DLINESIZE - 1
1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry
@@ -275,6 +275,30 @@ ENTRY(arm946_dma_flush_range)
mcr p15, 0, r0, c7, c10, 4 @ drain WB
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm946_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq arm946_dma_clean_range
+ bcs arm946_dma_inv_range
+ b arm946_dma_flush_range
+ENDPROC(arm946_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(arm946_dma_unmap_area)
+ mov pc, lr
+ENDPROC(arm946_dma_unmap_area)
+
ENTRY(arm946_cache_fns)
.long arm946_flush_kern_cache_all
.long arm946_flush_user_cache_all
@@ -282,8 +306,8 @@ ENTRY(arm946_cache_fns)
.long arm946_coherent_kern_range
.long arm946_coherent_user_range
.long arm946_flush_kern_dcache_area
- .long arm946_dma_inv_range
- .long arm946_dma_clean_range
+ .long arm946_dma_map_area
+ .long arm946_dma_unmap_area
.long arm946_dma_flush_range
diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S
index dbc39383e66a..53e632343849 100644
--- a/arch/arm/mm/proc-feroceon.S
+++ b/arch/arm/mm/proc-feroceon.S
@@ -274,7 +274,7 @@ ENTRY(feroceon_range_flush_kern_dcache_area)
* (same as v4wb)
*/
.align 5
-ENTRY(feroceon_dma_inv_range)
+feroceon_dma_inv_range:
tst r0, #CACHE_DLINESIZE - 1
bic r0, r0, #CACHE_DLINESIZE - 1
mcrne p15, 0, r0, c7, c10, 1 @ clean D entry
@@ -288,7 +288,7 @@ ENTRY(feroceon_dma_inv_range)
mov pc, lr
.align 5
-ENTRY(feroceon_range_dma_inv_range)
+feroceon_range_dma_inv_range:
mrs r2, cpsr
tst r0, #CACHE_DLINESIZE - 1
mcrne p15, 0, r0, c7, c10, 1 @ clean D entry
@@ -314,7 +314,7 @@ ENTRY(feroceon_range_dma_inv_range)
* (same as v4wb)
*/
.align 5
-ENTRY(feroceon_dma_clean_range)
+feroceon_dma_clean_range:
bic r0, r0, #CACHE_DLINESIZE - 1
1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry
add r0, r0, #CACHE_DLINESIZE
@@ -324,7 +324,7 @@ ENTRY(feroceon_dma_clean_range)
mov pc, lr
.align 5
-ENTRY(feroceon_range_dma_clean_range)
+feroceon_range_dma_clean_range:
mrs r2, cpsr
cmp r1, r0
subne r1, r1, #1 @ top address is inclusive
@@ -367,6 +367,44 @@ ENTRY(feroceon_range_dma_flush_range)
mcr p15, 0, r0, c7, c10, 4 @ drain WB
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(feroceon_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq feroceon_dma_clean_range
+ bcs feroceon_dma_inv_range
+ b feroceon_dma_flush_range
+ENDPROC(feroceon_dma_map_area)
+
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(feroceon_range_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq feroceon_range_dma_clean_range
+ bcs feroceon_range_dma_inv_range
+ b feroceon_range_dma_flush_range
+ENDPROC(feroceon_range_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(feroceon_dma_unmap_area)
+ mov pc, lr
+ENDPROC(feroceon_dma_unmap_area)
+
ENTRY(feroceon_cache_fns)
.long feroceon_flush_kern_cache_all
.long feroceon_flush_user_cache_all
@@ -374,8 +412,8 @@ ENTRY(feroceon_cache_fns)
.long feroceon_coherent_kern_range
.long feroceon_coherent_user_range
.long feroceon_flush_kern_dcache_area
- .long feroceon_dma_inv_range
- .long feroceon_dma_clean_range
+ .long feroceon_dma_map_area
+ .long feroceon_dma_unmap_area
.long feroceon_dma_flush_range
ENTRY(feroceon_range_cache_fns)
@@ -385,8 +423,8 @@ ENTRY(feroceon_range_cache_fns)
.long feroceon_coherent_kern_range
.long feroceon_coherent_user_range
.long feroceon_range_flush_kern_dcache_area
- .long feroceon_range_dma_inv_range
- .long feroceon_range_dma_clean_range
+ .long feroceon_range_dma_map_area
+ .long feroceon_dma_unmap_area
.long feroceon_range_dma_flush_range
.align 5
diff --git a/arch/arm/mm/proc-mohawk.S b/arch/arm/mm/proc-mohawk.S
index 9674d36cc97d..caa31154e7db 100644
--- a/arch/arm/mm/proc-mohawk.S
+++ b/arch/arm/mm/proc-mohawk.S
@@ -218,7 +218,7 @@ ENTRY(mohawk_flush_kern_dcache_area)
*
* (same as v4wb)
*/
-ENTRY(mohawk_dma_inv_range)
+mohawk_dma_inv_range:
tst r0, #CACHE_DLINESIZE - 1
mcrne p15, 0, r0, c7, c10, 1 @ clean D entry
tst r1, #CACHE_DLINESIZE - 1
@@ -241,7 +241,7 @@ ENTRY(mohawk_dma_inv_range)
*
* (same as v4wb)
*/
-ENTRY(mohawk_dma_clean_range)
+mohawk_dma_clean_range:
bic r0, r0, #CACHE_DLINESIZE - 1
1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry
add r0, r0, #CACHE_DLINESIZE
@@ -268,6 +268,30 @@ ENTRY(mohawk_dma_flush_range)
mcr p15, 0, r0, c7, c10, 4 @ drain WB
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(mohawk_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq mohawk_dma_clean_range
+ bcs mohawk_dma_inv_range
+ b mohawk_dma_flush_range
+ENDPROC(mohawk_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(mohawk_dma_unmap_area)
+ mov pc, lr
+ENDPROC(mohawk_dma_unmap_area)
+
ENTRY(mohawk_cache_fns)
.long mohawk_flush_kern_cache_all
.long mohawk_flush_user_cache_all
@@ -275,8 +299,8 @@ ENTRY(mohawk_cache_fns)
.long mohawk_coherent_kern_range
.long mohawk_coherent_user_range
.long mohawk_flush_kern_dcache_area
- .long mohawk_dma_inv_range
- .long mohawk_dma_clean_range
+ .long mohawk_dma_map_area
+ .long mohawk_dma_unmap_area
.long mohawk_dma_flush_range
ENTRY(cpu_mohawk_dcache_clean_area)
diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S
index 8e4f6dca8997..e5797f1c1db7 100644
--- a/arch/arm/mm/proc-xsc3.S
+++ b/arch/arm/mm/proc-xsc3.S
@@ -257,7 +257,7 @@ ENTRY(xsc3_flush_kern_dcache_area)
* - start - virtual start address
* - end - virtual end address
*/
-ENTRY(xsc3_dma_inv_range)
+xsc3_dma_inv_range:
tst r0, #CACHELINESIZE - 1
bic r0, r0, #CACHELINESIZE - 1
mcrne p15, 0, r0, c7, c10, 1 @ clean L1 D line
@@ -278,7 +278,7 @@ ENTRY(xsc3_dma_inv_range)
* - start - virtual start address
* - end - virtual end address
*/
-ENTRY(xsc3_dma_clean_range)
+xsc3_dma_clean_range:
bic r0, r0, #CACHELINESIZE - 1
1: mcr p15, 0, r0, c7, c10, 1 @ clean L1 D line
add r0, r0, #CACHELINESIZE
@@ -304,6 +304,30 @@ ENTRY(xsc3_dma_flush_range)
mcr p15, 0, r0, c7, c10, 4 @ data write barrier
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(xsc3_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq xsc3_dma_clean_range
+ bcs xsc3_dma_inv_range
+ b xsc3_dma_flush_range
+ENDPROC(xsc3_dma_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(xsc3_dma_unmap_area)
+ mov pc, lr
+ENDPROC(xsc3_dma_unmap_area)
+
ENTRY(xsc3_cache_fns)
.long xsc3_flush_kern_cache_all
.long xsc3_flush_user_cache_all
@@ -311,8 +335,8 @@ ENTRY(xsc3_cache_fns)
.long xsc3_coherent_kern_range
.long xsc3_coherent_user_range
.long xsc3_flush_kern_dcache_area
- .long xsc3_dma_inv_range
- .long xsc3_dma_clean_range
+ .long xsc3_dma_map_area
+ .long xsc3_dma_unmap_area
.long xsc3_dma_flush_range
ENTRY(cpu_xsc3_dcache_clean_area)
diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S
index 93df47265f2d..63037e2162f2 100644
--- a/arch/arm/mm/proc-xscale.S
+++ b/arch/arm/mm/proc-xscale.S
@@ -315,7 +315,7 @@ ENTRY(xscale_flush_kern_dcache_area)
* - start - virtual start address
* - end - virtual end address
*/
-ENTRY(xscale_dma_inv_range)
+xscale_dma_inv_range:
tst r0, #CACHELINESIZE - 1
bic r0, r0, #CACHELINESIZE - 1
mcrne p15, 0, r0, c7, c10, 1 @ clean D entry
@@ -336,7 +336,7 @@ ENTRY(xscale_dma_inv_range)
* - start - virtual start address
* - end - virtual end address
*/
-ENTRY(xscale_dma_clean_range)
+xscale_dma_clean_range:
bic r0, r0, #CACHELINESIZE - 1
1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry
add r0, r0, #CACHELINESIZE
@@ -363,6 +363,43 @@ ENTRY(xscale_dma_flush_range)
mcr p15, 0, r0, c7, c10, 4 @ Drain Write (& Fill) Buffer
mov pc, lr
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(xscale_dma_map_area)
+ add r1, r1, r0
+ cmp r2, #DMA_TO_DEVICE
+ beq xscale_dma_clean_range
+ bcs xscale_dma_inv_range
+ b xscale_dma_flush_range
+ENDPROC(xscale_dma_map_area)
+
+/*
+ * dma_map_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(xscale_dma_a0_map_area)
+ add r1, r1, r0
+ teq r2, #DMA_TO_DEVICE
+ beq xscale_dma_clean_range
+ b xscale_dma_flush_range
+ENDPROC(xscsale_dma_a0_map_area)
+
+/*
+ * dma_unmap_area(start, size, dir)
+ * - start - kernel virtual start address
+ * - size - size of region
+ * - dir - DMA direction
+ */
+ENTRY(xscale_dma_unmap_area)
+ mov pc, lr
+ENDPROC(xscale_dma_unmap_area)
+
ENTRY(xscale_cache_fns)
.long xscale_flush_kern_cache_all
.long xscale_flush_user_cache_all
@@ -370,8 +407,8 @@ ENTRY(xscale_cache_fns)
.long xscale_coherent_kern_range
.long xscale_coherent_user_range
.long xscale_flush_kern_dcache_area
- .long xscale_dma_inv_range
- .long xscale_dma_clean_range
+ .long xscale_dma_map_area
+ .long xscale_dma_unmap_area
.long xscale_dma_flush_range
/*
@@ -394,8 +431,8 @@ ENTRY(xscale_80200_A0_A1_cache_fns)
.long xscale_coherent_kern_range
.long xscale_coherent_user_range
.long xscale_flush_kern_dcache_area
- .long xscale_dma_flush_range
- .long xscale_dma_clean_range
+ .long xscale_dma_a0_map_area
+ .long xscale_dma_unmap_area
.long xscale_dma_flush_range
ENTRY(cpu_xscale_dcache_clean_area)