ARM: Tegra: Keep L2 available while MMU is on

As page tables can be outer cacheable we want to keep L2
available while MMU is on. Therefore, upon resuming from power
gating, enable L2 before MMU enable and upon power gating entry
disable L2 after MMU has been disabled. The optimization
is not stable with secure OS so leave the optimization out
for secure OS config. T148 has separate caches so there L2 flush
cannot be avoided. Also the caches are of different size so
the l2x0 module is initialized upon resume.

Bug 1046695

Change-Id: I520db89e880c08113e0b3e29a88efaad0c100045
Signed-off-by: Antti P Miettinen <amiettinen@nvidia.com>
Reviewed-on: http://git-master/r/204852
Reviewed-by: Automatic_Commit_Validation_User
GVS: Gerrit_Virtual_Submit
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>

8 files changed, 195 insertions, 54 deletions

diff --git a/arch/arm/mach-tegra/common.c b/arch/arm/mach-tegra/common.c
index 61362d17768c..2a76e6279969 100644
--- a/arch/arm/mach-tegra/common.c
+++ b/arch/arm/mach-tegra/common.c
@@ -470,30 +470,11 @@ void tegra_init_cache(bool init)
 #else
 #ifdef CONFIG_TEGRA_SILICON_PLATFORM
 	if (is_lp_cluster()) {
-#ifdef CONFIG_ARCH_TEGRA_14x_SOC
-		tag_latency = 0x110;
-		data_latency = 0x331;
-#else
-		tag_latency = 0x221;
-		data_latency = 0x221;
-#endif
+		tag_latency = tegra_cpu_c1_l2_tag_latency;
+		data_latency = tegra_cpu_c1_l2_data_latency;
 	} else {
-#ifdef CONFIG_ARCH_TEGRA_14x_SOC
-		tag_latency = 0x111;
-		data_latency = 0x441;
-#else
-		u32 speedo;
-		/* relax l2-cache latency for speedos 4,5,6 (T33's chips) */
-		speedo = tegra_cpu_speedo_id();
-		if (speedo == 4 || speedo == 5 || speedo == 6 ||
-		    speedo == 12 || speedo == 13) {
-			tag_latency = 0x442;
-			data_latency = 0x552;
-		} else {
-			tag_latency = 0x441;
-			data_latency = 0x551;
-		}
-#endif
+		tag_latency = tegra_cpu_c0_l2_tag_latency;
+		data_latency = tegra_cpu_c0_l2_data_latency;
 	}
 #else
 	tag_latency = 0x770;
@@ -712,6 +693,9 @@ void __init tegra20_init_early(void)
 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
 void __init tegra30_init_early(void)
 {
+	u32 speedo;
+	u32 tag_latency, data_latency;
+
 #ifndef CONFIG_SMP
 	/* For SMP system, initializing the reset handler here is too
 	   late. For non-SMP systems, the function that calls the reset
@@ -721,6 +705,27 @@ void __init tegra30_init_early(void)
 	tegra_apb_io_init();
 	tegra_perf_init();
 	tegra_init_fuse();
+	/*
+	 * Store G/LP cluster L2 latencies to IRAM and DRAM
+	 */
+	tegra_cpu_c1_l2_tag_latency = 0x221;
+	tegra_cpu_c1_l2_data_latency = 0x221;
+	writel_relaxed(0x221, tegra_cpu_c1_l2_tag_latency_iram);
+	writel_relaxed(0x221, tegra_cpu_c1_l2_data_latency_iram);
+	/* relax l2-cache latency for speedos 4,5,6 (T33's chips) */
+	speedo = tegra_cpu_speedo_id();
+	if (speedo == 4 || speedo == 5 || speedo == 6 ||
+	    speedo == 12 || speedo == 13) {
+		tag_latency = 0x442;
+		data_latency = 0x552;
+	} else {
+		tag_latency = 0x441;
+		data_latency = 0x551;
+	}
+	tegra_cpu_c0_l2_tag_latency = tag_latency;
+	tegra_cpu_c0_l2_data_latency = data_latency;
+	writel_relaxed(tag_latency, tegra_cpu_c0_l2_tag_latency_iram);
+	writel_relaxed(data_latency, tegra_cpu_c0_l2_data_latency_iram);
 	tegra_init_cache(true);
 	tegra_pmc_init();
 	tegra_powergate_init();
@@ -780,6 +785,14 @@ void __init tegra14x_init_early(void)
 	tegra14x_init_dvfs();
 	tegra_common_init_clock();
 	tegra_clk_init_from_table(tegra14x_clk_init_table);
+	tegra_cpu_c1_l2_tag_latency = 0x110;
+	tegra_cpu_c1_l2_data_latency = 0x331;
+	writel_relaxed(0x110, tegra_cpu_c1_l2_tag_latency_iram);
+	writel_relaxed(0x331, tegra_cpu_c1_l2_data_latency_iram);
+	tegra_cpu_c0_l2_tag_latency = 0x111;
+	tegra_cpu_c0_l2_data_latency = 0x441;
+	writel_relaxed(0x111, tegra_cpu_c0_l2_tag_latency_iram);
+	writel_relaxed(0x441, tegra_cpu_c0_l2_data_latency_iram);
 	tegra_init_cache(true);
 	tegra_pmc_init();
 	tegra_powergate_init();
diff --git a/arch/arm/mach-tegra/headsmp.S b/arch/arm/mach-tegra/headsmp.S
index 72b1fc778ae1..4a39cc63d181 100644
--- a/arch/arm/mach-tegra/headsmp.S
+++ b/arch/arm/mach-tegra/headsmp.S
@@ -24,6 +24,7 @@
 #include <asm/assembler.h>
 #include <asm/cache.h>
 #include <asm/page.h>
+#include <asm/hardware/cache-l2x0.h>
 
 #include "flowctrl.h"
 #include "iomap.h"
@@ -35,6 +36,9 @@
 
 #define DEBUG_CPU_RESET_HANDLER	0	/* Non-zero enables debug code */
 
+#define RESET_DATA_PHYS (TEGRA_RESET_HANDLER_BASE \
+	+ __tegra_cpu_reset_handler_data - __tegra_cpu_reset_handler_start)
+
 #define RESET_DATA(x)	((TEGRA_RESET_##x)*4)
 
 #ifdef CONFIG_SMP
@@ -100,7 +104,7 @@ ENTRY(tegra_resume)
 	movw	r0, #0x3FFD	@ enable, enable_ext, cluster_switch, immed, & bitmaps
 	bic	r1, r1, r0
 	str	r1, [r2]
-#endif
+#endif /* !CONFIG_ARCH_TEGRA_2x_SOC */
 
 #if defined(CONFIG_HAVE_ARM_SCU)
 	/* enable SCU */
@@ -108,7 +112,7 @@ ENTRY(tegra_resume)
 	ldr	r1, [r0]
 	orr	r1, r1, #1
 	str	r1, [r0]
-#endif
+#endif /* CONFIG_HAVE_ARM_SCU */
 
 #ifdef CONFIG_TRUSTED_FOUNDATIONS
 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
@@ -117,15 +121,73 @@ ENTRY(tegra_resume)
 	adr	r1, tegra_resume_smc_entry_time
 	str	r0, [r1]
 
-	/* wake up (should have specified args?) */
+	/* wake up */
+	mov	r0, #0x00000003
 	bl	tegra_generic_smc
 
 	mov32	r1, TEGRA_TMRUS_BASE
 	ldr	r0, [r1]
 	adr	r1, tegra_resume_smc_exit_time
 	str	r0, [r1]
+#endif /* !CONFIG_ARCH_TEGRA_11x_SOC */
+#endif /* CONFIG_TRUSTED_FOUNDATIONS */
+
+#ifdef CONFIG_CACHE_L2X0
+#if !defined(CONFIG_TRUSTED_FOUNDATIONS) && !defined(CONFIG_ARCH_TEGRA_14x_SOC)
+	adr	r0, tegra_resume_l2_init
+	ldr	r1, [r0]
+	tst	r1, #1
+	beq	no_l2_init
+	/* Enable L2 */
+	bic	r1, #1
+	str	r1, [r0]
+	mov32	r3, TEGRA_ARM_PL310_BASE
+#if defined(CONFIG_ARCH_TEGRA_2x_SOC)
+	mov32	r0, 0x331			/* tag latency */
+	mov32	r1, 0x441			/* data latency */
+#elif defined(CONFIG_ARCH_TEGRA_3x_SOC) || defined(CONFIG_ARCH_TEGRA_14x_SOC)
+#ifdef CONFIG_TEGRA_SILICON_PLATFORM
+	mov32	r0, TEGRA_FLOW_CTRL_BASE + 0x2c	/* FLOW_CTRL_CLUSTER_CONTROL */
+	mov32	r2, RESET_DATA_PHYS
+	ldr	r1, [r0]
+	tst	r1, #1				/* 0 == G, 1 == LP */
+	ldrne	r0, [r2, #RESET_DATA(C1_L2_TAG_LATENCY)]
+	ldrne	r1, [r2, #RESET_DATA(C1_L2_DATA_LATENCY)]
+	ldreq	r0, [r2, #RESET_DATA(C0_L2_TAG_LATENCY)]
+	ldreq	r1, [r2, #RESET_DATA(C0_L2_DATA_LATENCY)]
+#else /* !CONFIG_TEGRA_SILICON_PLATFORM */
+	mov32	r0, #0x770			/* tag latency */
+	mov32	r1, #0x770			/* data latency */
+#endif /* ?CONFIG_TEGRA_SILICON_PLATFORM */
+#endif /* CONFIG_ARCH_TEGRA_3x_SOC || CONFIG_ARCH_TEGRA_14x_SOC */
+	str	r0, [r3, #L2X0_TAG_LATENCY_CTRL]
+	str	r1, [r3, #L2X0_DATA_LATENCY_CTRL]
+#ifndef CONFIG_TEGRA_FPGA_PLATFORM
+#ifdef CONFIG_ARCH_TEGRA_14x_SOC
+	mov32	r0, 0x40000007	/* Enable double line fill */
+#else
+	mov	r0, #7
 #endif
-#endif
+	str	r0, [r3, #L2X0_PREFETCH_CTRL]
+#endif /* !CONFIG_TEGRA_FPGA_PLATFORM */
+	mov	r0, #3
+	str	r0, [r3, #L2X0_POWER_CTRL]
+	/* figure out aux ctrl */
+	ldr	r2, [r3, #L2X0_CACHE_TYPE]
+	and	r2, r2, #0x700
+	lsl	r2, r2, #(17-8)
+	mov32	r4, 0x7C400001
+	orr	r2, r2, r4
+	ldr	r4, [r3, #L2X0_AUX_CTRL]
+	mov32	r5, 0x8200c3fe
+	and	r4, r4, r5
+	orr	r2, r2, r4
+	str	r2, [r3, #L2X0_AUX_CTRL]
+	mov	r2, #1
+	str	r2, [r3, #L2X0_CTRL]
+#endif /* ?CONFIG_TRUSTED_FOUNDATIONS */
+#endif /* CONFIG_CACHE_L2X0 */
+no_l2_init:
 	b	cpu_resume
 ENDPROC(tegra_resume)
 
@@ -134,7 +196,7 @@ ENDPROC(tegra_resume)
 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
 	.globl tegra_resume_smc_entry_time
 	.globl tegra_resume_smc_exit_time
-#endif
+#endif /* !CONFIG_ARCH_TEGRA_11x_SOC */
 	.globl tegra_resume_entry_time
 	.globl tegra_resume_timestamps_end
 tegra_resume_timestamps_start:
@@ -143,13 +205,17 @@ tegra_resume_smc_entry_time:
 	.long	0
 tegra_resume_smc_exit_time:
 	.long	0
-#endif
+#endif /* !CONFIG_ARCH_TEGRA_11x_SOC */
 tegra_resume_entry_time:
 	.long	0
 tegra_resume_timestamps_end:
-ENTRY(__tegra_resume_timestamps_end)
-#endif
-#endif
+#endif /* CONFIG_TRUSTED_FOUNDATIONS */
+#ifdef CONFIG_CACHE_L2X0
+	.globl tegra_resume_l2_init
+tegra_resume_l2_init:
+	.long 0
+#endif /* CONFIG_CACHE_L2X0 */
+#endif /* CONFIG_PM_SLEEP */
 
 /*
  *	__invalidate_cpu_state
@@ -448,6 +514,11 @@ __tegra_cpu_reset_handler_data:
 	.rept	TEGRA_RESET_DATA_SIZE
 	.long	0
 	.endr
-	.size	__tegra_cpu_reset_handler_data, . - __tegra_cpu_reset_handler_data
+	.size	__tegra_cpu_reset_handler_data, \
+	. - __tegra_cpu_reset_handler_data
 	.align L1_CACHE_SHIFT
 ENTRY(__tegra_cpu_reset_handler_end)
+
+	.globl	__tegra_cpu_reset_handler_data_offset
+	.equ	__tegra_cpu_reset_handler_data_offset, \
+	__tegra_cpu_reset_handler_data - __tegra_cpu_reset_handler_start
diff --git a/arch/arm/mach-tegra/pm.c b/arch/arm/mach-tegra/pm.c
index 8078b578d72b..f49a8ac6c2cb 100644
--- a/arch/arm/mach-tegra/pm.c
+++ b/arch/arm/mach-tegra/pm.c
@@ -599,9 +599,6 @@ unsigned int tegra_idle_power_down_last(unsigned int sleep_time,
 {
 	u32 reg;
 	unsigned int remain;
-#if defined(CONFIG_CACHE_L2X0) && !defined(CONFIG_ARCH_TEGRA_14x_SOC)
-	pgd_t *pgd;
-#endif
 
 	/* Only the last cpu down does the final suspend steps */
 	reg = readl(pmc + PMC_CTRL);
@@ -673,27 +670,23 @@ unsigned int tegra_idle_power_down_last(unsigned int sleep_time,
 	cpu_cluster_pm_enter();
 	suspend_cpu_complex(flags);
 	tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
-#ifdef CONFIG_CACHE_L2X0
+#if defined(CONFIG_CACHE_L2X0)
+#if defined(CONFIG_TRUSTED_FOUNDATIONS)
 	flush_cache_all();
-#ifdef CONFIG_ARCH_TEGRA_14x_SOC
-	outer_flush_all();
-#else
-	/*
-	 * No need to flush complete L2. Cleaning kernel and IO mappings
-	 * is enough for the LP code sequence that has L2 disabled but
-	 * MMU on.
-	 */
-	pgd = cpu_get_pgd();
-	outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
-			  __pa(pgd + PTRS_PER_PGD));
-#endif
 	outer_disable();
+#elif !defined(CONFIG_ARCH_TEGRA_14x_SOC)
+	tegra_resume_l2_init = 1;
+	__cpuc_flush_dcache_area(&tegra_resume_l2_init, sizeof(unsigned long));
+	outer_flush_range(__pa(&tegra_resume_l2_init),
+			  __pa(&tegra_resume_l2_init) + sizeof(unsigned long));
 #endif
+#endif
+
 	tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
 
-#ifdef CONFIG_ARCH_TEGRA_14x_SOC
+#if defined(CONFIG_ARCH_TEGRA_14x_SOC)
 	tegra_init_cache(true);
-#else
+#elif defined(CONFIG_TRUSTED_FOUNDATIONS)
 	tegra_init_cache(false);
 #endif
 
@@ -1048,7 +1041,6 @@ int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
 #endif
 
 	flush_cache_all();
-	outer_flush_all();
 	outer_disable();
 
 	if (mode == TEGRA_SUSPEND_LP2)
diff --git a/arch/arm/mach-tegra/reset.h b/arch/arm/mach-tegra/reset.h
index 44a671160e57..413fdfbab082 100644
--- a/arch/arm/mach-tegra/reset.h
+++ b/arch/arm/mach-tegra/reset.h
@@ -25,7 +25,11 @@
 #define TEGRA_RESET_STARTUP_SECONDARY	3
 #define TEGRA_RESET_STARTUP_LP2		4
 #define TEGRA_RESET_STARTUP_LP1		5
-#define TEGRA_RESET_DATA_SIZE		6
+#define TEGRA_RESET_C0_L2_TAG_LATENCY	6
+#define TEGRA_RESET_C0_L2_DATA_LATENCY	7
+#define TEGRA_RESET_C1_L2_TAG_LATENCY	8
+#define TEGRA_RESET_C1_L2_DATA_LATENCY	9
+#define TEGRA_RESET_DATA_SIZE		10
 
 #ifndef __ASSEMBLY__
 
@@ -52,6 +56,38 @@ void tegra_secondary_startup(void);
 		 (u32)__tegra_cpu_reset_handler_start))))
 #endif
 
+#define tegra_cpu_c0_l2_tag_latency \
+	__tegra_cpu_reset_handler_data[TEGRA_RESET_C0_L2_TAG_LATENCY]
+
+#define tegra_cpu_c0_l2_tag_latency_iram \
+	((u32 *)(IO_ADDRESS(TEGRA_RESET_HANDLER_BASE +			\
+	((u32)&__tegra_cpu_reset_handler_data[TEGRA_RESET_C0_L2_TAG_LATENCY] \
+	 - (u32)__tegra_cpu_reset_handler_start))))
+
+#define tegra_cpu_c0_l2_data_latency \
+	__tegra_cpu_reset_handler_data[TEGRA_RESET_C0_L2_DATA_LATENCY]
+
+#define tegra_cpu_c0_l2_data_latency_iram \
+	((u32 *)(IO_ADDRESS(TEGRA_RESET_HANDLER_BASE +			\
+	((u32)&__tegra_cpu_reset_handler_data[TEGRA_RESET_C0_L2_DATA_LATENCY] \
+	 - (u32)__tegra_cpu_reset_handler_start))))
+
+#define tegra_cpu_c1_l2_tag_latency \
+	__tegra_cpu_reset_handler_data[TEGRA_RESET_C1_L2_TAG_LATENCY]
+
+#define tegra_cpu_c1_l2_tag_latency_iram \
+	((u32 *)(IO_ADDRESS(TEGRA_RESET_HANDLER_BASE +			\
+	((u32)&__tegra_cpu_reset_handler_data[TEGRA_RESET_C1_L2_TAG_LATENCY] \
+	 - (u32)__tegra_cpu_reset_handler_start))))
+
+#define tegra_cpu_c1_l2_data_latency \
+	__tegra_cpu_reset_handler_data[TEGRA_RESET_C1_L2_DATA_LATENCY]
+
+#define tegra_cpu_c1_l2_data_latency_iram \
+	((u32 *)(IO_ADDRESS(TEGRA_RESET_HANDLER_BASE +			\
+	((u32)&__tegra_cpu_reset_handler_data[TEGRA_RESET_C1_L2_DATA_LATENCY] \
+	 - (u32)__tegra_cpu_reset_handler_start))))
+
 #define tegra_cpu_reset_handler_offset \
 		((u32)__tegra_cpu_reset_handler - \
 		 (u32)__tegra_cpu_reset_handler_start)
diff --git a/arch/arm/mach-tegra/sleep-t20.S b/arch/arm/mach-tegra/sleep-t20.S
index b189e573ecdb..ca540dd51599 100644
--- a/arch/arm/mach-tegra/sleep-t20.S
+++ b/arch/arm/mach-tegra/sleep-t20.S
@@ -177,6 +177,7 @@ ENTRY(tegra2_sleep_core_finish)
 	sub	r1, r1, r2
 	mov32	r2, TEGRA_IRAM_CODE_AREA
 	add	r1, r1, r2
+	mov	r11, #0
 	b	tegra_turn_off_mmu
 ENDPROC(tegra2_sleep_core_finish)
 
diff --git a/arch/arm/mach-tegra/sleep-t30.S b/arch/arm/mach-tegra/sleep-t30.S
index 47874ba86156..41c524e42daf 100644
--- a/arch/arm/mach-tegra/sleep-t30.S
+++ b/arch/arm/mach-tegra/sleep-t30.S
@@ -266,6 +266,7 @@ ENTRY(tegra3_sleep_core_finish)
 	sub	r1, r1, r2
 	mov32	r2, TEGRA_IRAM_CODE_AREA
 	add	r1, r1, r2
+	mov	r11, #0
 	b	tegra_turn_off_mmu
 ENDPROC(tegra3_sleep_core_finish)
 
diff --git a/arch/arm/mach-tegra/sleep.S b/arch/arm/mach-tegra/sleep.S
index b634d0e75927..0fa40cc407dd 100644
--- a/arch/arm/mach-tegra/sleep.S
+++ b/arch/arm/mach-tegra/sleep.S
@@ -36,11 +36,13 @@
 #include <asm/glue-cache.h>
 #include <asm/glue-proc.h>
 #include <asm/cp15.h>
+#include <asm/hardware/cache-l2x0.h>
 
 #include "iomap.h"
 
 #include "sleep.h"
 #include "flowctrl.h"
+#include "reset.h"
 
 #define CLK_RESET_CCLK_BURST		0x20
 #define CLK_RESET_CCLK_DIVIDER		0x24
@@ -230,6 +232,7 @@ ENTRY(tegra_sleep_cpu_finish)
 	mov32	r1, tegra3_tear_down_cpu
 #endif
 	add	r1, r1, r0
+	mov	r11, #1
 	b	tegra_turn_off_mmu
 ENDPROC(tegra_sleep_cpu_finish)
 
@@ -238,6 +241,7 @@ ENDPROC(tegra_sleep_cpu_finish)
  *
  * r0 = v2p
  * r1 = physical address to jump to with mmu off
+ * r11 = L2 disable/flush
  */
 ENTRY(tegra_turn_off_mmu)
 	/*
@@ -275,6 +279,7 @@ tegra_pgd_phys_address:
  * tegra_shut_off_mmu
  *
  * r0 = physical address to jump to with mmu off
+ * r11 = L2 disable/flush
  *
  * called with VA=PA mapping
  * turns off MMU, icache, dcache and branch prediction
@@ -287,7 +292,26 @@ tegra_shut_off_mmu:
 	dsb
 	mcr	p15, 0, r3, c1, c0, 0
 	isb
-	mov	pc, r0
+#if defined(CONFIG_CACHE_L2X0) && !defined(CONFIG_TRUSTED_FOUNDATIONS)
+	tst	r11, #1
+	beq	2f
+	mov32	r1, TEGRA_ARM_PL310_BASE
+#ifdef CONFIG_ARCH_TEGRA_14x_SOC
+	/* need to flush the L2 */
+	ldr	r2, [r1, #L2X0_AUX_CTRL]
+	mov	r2, #0xff
+	tst	r2, #(1 << 16)			@ associativity
+	orrne	r2, #0xff00
+	str	r2, [r1, #L2X0_CLEAN_INV_WAY]
+1:	ldr	r3, [r1, #L2X0_CLEAN_INV_WAY]
+	tst	r3, r2
+	bne	1b
+#endif /* CONFIG_ARCH_TEGRA_14x_SOC */
+	/* Disable L2 */
+	mov	r2, #0
+	str	r2, [r1, #L2X0_CTRL]
+#endif /* CONFIG_CACHE_L2X0 && !CONFIG_TRUSTED_FOUNDATIONS */
+2:	mov	pc, r0
 
 /*
  * tegra_cpu_clk32k
diff --git a/arch/arm/mach-tegra/sleep.h b/arch/arm/mach-tegra/sleep.h
index 5e72ade42bf8..1311acfcdf2c 100644
--- a/arch/arm/mach-tegra/sleep.h
+++ b/arch/arm/mach-tegra/sleep.h
@@ -328,6 +328,9 @@ extern unsigned long tegra_resume_smc_exit_time;
 #endif
 extern unsigned long tegra_resume_entry_time;
 #endif
+#if defined(CONFIG_CACHE_L2X0) && defined(CONFIG_PM_SLEEP)
+extern unsigned long tegra_resume_l2_init;
+#endif
 
 static inline void *tegra_iram_start(void)
 {