ARM: tegra: power: Add Tegra3 balanced throttling

Balanced CPU and core domains thermal throttling on Tegra3. When
throttling is enabled the new algorithm caps core bus frequencies
(EMC, cbus and sbus) along with CPU rate. The throttling steps, and
time spent on each step are pre-defined based on characterization
results.

Change-Id: Id18109f6319032aad4332b13955996116a7f6485
Reviewed-on: http://git-master/r/51497
Reviewed-by: Varun Colbert <vcolbert@nvidia.com>
Tested-by: Varun Colbert <vcolbert@nvidia.com>

4 files changed, 291 insertions, 4 deletions

diff --git a/arch/arm/mach-tegra/Makefile b/arch/arm/mach-tegra/Makefile
index 04c6ef50dda3..1dadedc81857 100644
--- a/arch/arm/mach-tegra/Makefile
+++ b/arch/arm/mach-tegra/Makefile
@@ -82,7 +82,8 @@ obj-$(CONFIG_ARCH_TEGRA_2x_SOC)         += cpuidle-t2.o
 obj-$(CONFIG_ARCH_TEGRA_3x_SOC)         += cpuidle-t3.o
 endif
 ifeq ($(CONFIG_TEGRA_THERMAL_THROTTLE),y)
-obj-y                                   += tegra2-throttle.o
+obj-$(CONFIG_ARCH_TEGRA_2x_SOC)         += tegra2-throttle.o
+obj-$(CONFIG_ARCH_TEGRA_3x_SOC)         += tegra3-throttle.o
 endif
 obj-$(CONFIG_TEGRA_IOVMM)               += iovmm.o
 obj-$(CONFIG_TEGRA_IOVMM_GART)          += iovmm-gart.o
diff --git a/arch/arm/mach-tegra/dvfs.h b/arch/arm/mach-tegra/dvfs.h
index 692cd09f1076..cc9983a623fe 100644
--- a/arch/arm/mach-tegra/dvfs.h
+++ b/arch/arm/mach-tegra/dvfs.h
@@ -112,6 +112,8 @@ void tegra_dvfs_rail_on(struct dvfs_rail *rail, ktime_t now);
 void tegra_dvfs_rail_pause(struct dvfs_rail *rail, ktime_t delta, bool on);
 struct dvfs_rail *tegra_dvfs_get_rail_by_name(const char *reg_id);
 int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate);
+void tegra_dvfs_core_cap_enable(bool enable);
+void tegra_dvfs_core_cap_level_set(int level);
 #else
 static inline void tegra_soc_init_dvfs(void)
 {}
@@ -142,6 +144,10 @@ static inline struct dvfs_rail *tegra_dvfs_get_rail_by_name(const char *reg_id)
 { return NULL;}
 static inline int tegra_dvfs_predict_millivolts(struct clk *c, unsigned long rate)
 { return 0; }
+static inline void tegra_dvfs_core_cap_enable(bool enable)
+{}
+static inline void tegra_dvfs_core_cap_level_set(int level)
+{}
 #endif
 
 #endif
diff --git a/arch/arm/mach-tegra/tegra3-throttle.c b/arch/arm/mach-tegra/tegra3-throttle.c
new file mode 100644
index 000000000000..4be94a86450d
--- /dev/null
+++ b/arch/arm/mach-tegra/tegra3-throttle.c
@@ -0,0 +1,280 @@
+/*
+ * arch/arm/mach-tegra/tegra3_throttle.c
+ *
+ * Copyright (c) 2011, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include <linux/kernel.h>
+#include <linux/cpufreq.h>
+#include <linux/delay.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/clk.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/uaccess.h>
+
+#include "clock.h"
+#include "cpu-tegra.h"
+#include "dvfs.h"
+
+/* tegra throttling require frequencies in the table to be in ascending order */
+static struct cpufreq_frequency_table *cpu_freq_table;
+static struct mutex *cpu_throttle_lock;
+
+static struct {
+	unsigned int cpu_freq;
+	int core_cap_level;
+	int ms;
+} throttle_table[] = {
+	{      0, 1000, 2000 },	/* placeholder for cpu floor rate */
+	{ 640000, 1000, 10000 },
+	{ 760000, 1000, 2000 },
+	{ 760000, 1050, 2000 },
+	{1000000, 1050, 2000 },
+	{1000000, 1100, 2000 },
+};
+
+static int is_throttling;
+static int throttle_index;
+static struct delayed_work throttle_work;
+static struct workqueue_struct *workqueue;
+static DEFINE_MUTEX(tegra_throttle_lock);
+
+static unsigned int clip_to_table(unsigned int cpu_freq)
+{
+	int i;
+
+	for (i = 0; cpu_freq_table[i].frequency != CPUFREQ_TABLE_END; i++) {
+		if (cpu_freq_table[i].frequency > cpu_freq)
+			break;
+	}
+	i = (i == 0) ? 0 : i-1;
+	return cpu_freq_table[i].frequency;
+}
+
+static void tegra_throttle_work_func(struct work_struct *work)
+{
+	unsigned int cpu_freq;
+	int core_level;
+
+	mutex_lock(cpu_throttle_lock);
+	if (!is_throttling) {
+		mutex_unlock(cpu_throttle_lock);
+		return;
+	}
+
+	cpu_freq = tegra_getspeed(0);
+	throttle_index -= throttle_index ? 1 : 0;
+
+	core_level = throttle_table[throttle_index].core_cap_level;
+	if (throttle_table[throttle_index].cpu_freq < cpu_freq)
+		tegra_cpu_set_speed_cap(NULL);
+
+	if (throttle_index || (throttle_table[0].cpu_freq < cpu_freq))
+		queue_delayed_work(workqueue, &throttle_work,
+			msecs_to_jiffies(throttle_table[throttle_index].ms));
+
+	mutex_unlock(cpu_throttle_lock);
+
+	tegra_dvfs_core_cap_level_set(core_level);
+}
+
+/*
+ * tegra_throttling_enable
+ * This function may sleep
+ */
+void tegra_throttling_enable(bool enable)
+{
+	mutex_lock(&tegra_throttle_lock);
+	mutex_lock(cpu_throttle_lock);
+
+	if (enable && !(is_throttling++)) {
+		int core_level;
+		unsigned int cpu_freq = tegra_getspeed(0);
+		throttle_index = ARRAY_SIZE(throttle_table) - 1;
+
+		core_level = throttle_table[throttle_index].core_cap_level;
+		if (throttle_table[throttle_index].cpu_freq < cpu_freq)
+			tegra_cpu_set_speed_cap(NULL);
+
+		queue_delayed_work(workqueue, &throttle_work,
+			msecs_to_jiffies(throttle_table[throttle_index].ms));
+
+		mutex_unlock(cpu_throttle_lock);
+
+		tegra_dvfs_core_cap_level_set(core_level);
+		tegra_dvfs_core_cap_enable(true);
+
+		mutex_unlock(&tegra_throttle_lock);
+		return;
+	}
+
+	if (!enable && is_throttling) {
+		if (!(--is_throttling)) {
+			/* restore speed requested by governor */
+			tegra_cpu_set_speed_cap(NULL);
+			mutex_unlock(cpu_throttle_lock);
+
+			tegra_dvfs_core_cap_enable(false);
+			cancel_delayed_work_sync(&throttle_work);
+			mutex_unlock(&tegra_throttle_lock);
+			return;
+		}
+	}
+
+	mutex_unlock(cpu_throttle_lock);
+	mutex_unlock(&tegra_throttle_lock);
+}
+EXPORT_SYMBOL_GPL(tegra_throttling_enable);
+
+unsigned int tegra_throttle_governor_speed(unsigned int requested_speed)
+{
+	return is_throttling ?
+		min(requested_speed, throttle_table[throttle_index].cpu_freq) :
+		requested_speed;
+}
+
+bool tegra_is_throttling(void)
+{
+	return is_throttling;
+}
+
+int __init tegra_throttle_init(struct mutex *cpu_lock)
+{
+	int i;
+	struct tegra_cpufreq_table_data *table_data =
+		tegra_cpufreq_table_get();
+	if (IS_ERR_OR_NULL(table_data))
+		return -EINVAL;
+
+	/*
+	 * High-priority, others flags default: not bound to a specific
+	 * CPU, has rescue worker task (in case of allocation deadlock,
+	 * etc.).  Single-threaded.
+	 */
+	workqueue = alloc_workqueue("cpu-tegra",
+				    WQ_HIGHPRI | WQ_UNBOUND | WQ_RESCUER, 1);
+	if (!workqueue)
+		return -ENOMEM;
+	INIT_DELAYED_WORK(&throttle_work, tegra_throttle_work_func);
+
+	cpu_throttle_lock = cpu_lock;
+	cpu_freq_table = table_data->freq_table;
+	throttle_table[0].cpu_freq =
+		cpu_freq_table[table_data->throttle_lowest_index].frequency;
+
+	for (i = 0; i < ARRAY_SIZE(throttle_table); i++) {
+		unsigned int cpu_freq = throttle_table[i].cpu_freq;
+		throttle_table[i].cpu_freq = clip_to_table(cpu_freq);
+	}
+
+	return 0;
+}
+
+void tegra_throttle_exit(void)
+{
+	destroy_workqueue(workqueue);
+}
+
+#ifdef CONFIG_DEBUG_FS
+
+static int throttle_debug_set(void *data, u64 val)
+{
+	tegra_throttling_enable(val);
+	return 0;
+}
+static int throttle_debug_get(void *data, u64 *val)
+{
+	*val = (u64) is_throttling;
+	return 0;
+}
+DEFINE_SIMPLE_ATTRIBUTE(throttle_fops, throttle_debug_get, throttle_debug_set,
+			"%llu\n");
+static int table_show(struct seq_file *s, void *data)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(throttle_table); i++)
+		seq_printf(s, "[%d] = %7u %4d %5d\n",
+			i, throttle_table[i].cpu_freq,
+			throttle_table[i].core_cap_level, throttle_table[i].ms);
+	return 0;
+}
+
+static int table_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, table_show, inode->i_private);
+}
+
+static ssize_t table_write(struct file *file,
+	const char __user *userbuf, size_t count, loff_t *ppos)
+{
+	char buf[80];
+	int table_idx;
+	unsigned int cpu_freq;
+	int core_cap_level;
+	int ms;
+
+	if (sizeof(buf) <= count)
+		return -EINVAL;
+
+	if (copy_from_user(buf, userbuf, count))
+		return -EFAULT;
+
+	/* terminate buffer and trim - white spaces may be appended
+	 *  at the end when invoked from shell command line */
+	buf[count] = '\0';
+	strim(buf);
+
+	if (sscanf(buf, "[%d] = %u %d %d",
+		   &table_idx, &cpu_freq, &core_cap_level, &ms) != 4)
+		return -1;
+
+	if ((table_idx < 0) || (table_idx >= ARRAY_SIZE(throttle_table)))
+		return -EINVAL;
+
+	/* round new settings before updating table */
+	throttle_table[table_idx].cpu_freq = clip_to_table(cpu_freq);
+	throttle_table[table_idx].core_cap_level = (core_cap_level / 50) * 50;
+	throttle_table[table_idx].ms = jiffies_to_msecs(msecs_to_jiffies(ms));
+
+	return count;
+}
+
+static const struct file_operations table_fops = {
+	.open		= table_open,
+	.read		= seq_read,
+	.write		= table_write,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+
+int __init tegra_throttle_debug_init(struct dentry *cpu_tegra_debugfs_root)
+{
+	if (!debugfs_create_file("throttle", 0644, cpu_tegra_debugfs_root,
+				 NULL, &throttle_fops))
+		return -ENOMEM;
+
+	if (!debugfs_create_file("throttle_table", 0644, cpu_tegra_debugfs_root,
+				 NULL, &table_fops))
+		return -ENOMEM;
+
+	return 0;
+}
+#endif /* CONFIG_DEBUG_FS */
+
diff --git a/arch/arm/mach-tegra/tegra3_clocks.c b/arch/arm/mach-tegra/tegra3_clocks.c
index dd8eec59dcd0..8fc8fde200ba 100644
--- a/arch/arm/mach-tegra/tegra3_clocks.c
+++ b/arch/arm/mach-tegra/tegra3_clocks.c
@@ -4167,9 +4167,9 @@ static struct cpufreq_frequency_table freq_table_1p4GHz[] = {
 
 static struct tegra_cpufreq_table_data cpufreq_tables[] = {
 	{ freq_table_300MHz, 0, 1 },
-	{ freq_table_1p0GHz, 2, 7, 2},
-	{ freq_table_1p3GHz, 2, 9, 2},
-	{ freq_table_1p4GHz, 2, 10, 2},
+	{ freq_table_1p0GHz, 1, 7, 2},
+	{ freq_table_1p3GHz, 1, 9, 2},
+	{ freq_table_1p4GHz, 1, 10, 2},
 };
 
 static int clip_cpu_rate_limits(