From a1c9f879ab8da0c4fc0a9c75791de5ada0a2ffba Mon Sep 17 00:00:00 2001 From: Sai Charan Gurrappadi Date: Wed, 25 Jul 2012 08:37:14 -0700 Subject: cpuquiet: Runnable threads governor [perf] The runnable threads governor only looks at the average number of runnables in the system to make a decision when bringing cores offline/online. First pass; tweaks thresholds and delays to reduce decision latency to about ~50-70ms per core (from ~100-150ms per core) Change-Id: Idd3b268a74a8f56ad3fc0e5c7f388174d1b6611f Signed-off-by: Sai Charan Gurrappadi Reviewed-on: http://git-master/r/124679 Reviewed-by: Automatic_Commit_Validation_User Reviewed-by: Diwakar Tundlam --- drivers/cpuquiet/governors/Makefile | 2 +- drivers/cpuquiet/governors/runnable_threads.c | 247 ++++++++++++++++++++++++++ include/linux/sched.h | 1 + kernel/sched/core.c | 12 ++ 4 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 drivers/cpuquiet/governors/runnable_threads.c diff --git a/drivers/cpuquiet/governors/Makefile b/drivers/cpuquiet/governors/Makefile index c70803127082..e199d73008f1 100644 --- a/drivers/cpuquiet/governors/Makefile +++ b/drivers/cpuquiet/governors/Makefile @@ -1 +1 @@ -obj-y += userspace.o balanced.o +obj-y += userspace.o balanced.o runnable_threads.o diff --git a/drivers/cpuquiet/governors/runnable_threads.c b/drivers/cpuquiet/governors/runnable_threads.c new file mode 100644 index 000000000000..a44abe9b25f2 --- /dev/null +++ b/drivers/cpuquiet/governors/runnable_threads.c @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2012 NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef enum { + DISABLED, + IDLE, + DOWN, + UP, +} RUNNABLES_STATE; + +static struct delayed_work runnables_work; +static struct kobject *runnables_kobject; + +/* configurable parameters */ +static unsigned int sample_rate = 20; /* msec */ + +static RUNNABLES_STATE runnables_state; +static struct workqueue_struct *runnables_wq; + +#define NR_FSHIFT_EXP 3 +#define NR_FSHIFT (1 << NR_FSHIFT_EXP) +/* avg run threads * 8 (e.g., 11 = 1.375 threads) */ +static unsigned int default_thresholds[] = { + 9, 17, 25, UINT_MAX +}; + +static unsigned int nr_run_last; +static unsigned int nr_run_hysteresis = 4; /* 1 / 4 thread */ +static unsigned int default_threshold_level = 4; /* 1 / 4 thread */ +static unsigned int nr_run_thresholds[NR_CPUS]; + +DEFINE_MUTEX(runnables_work_lock); + +static void update_runnables_state(void) +{ + unsigned int nr_cpus = num_online_cpus(); + int max_cpus = pm_qos_request(PM_QOS_MAX_ONLINE_CPUS) ? : 4; + int min_cpus = pm_qos_request(PM_QOS_MIN_ONLINE_CPUS); + unsigned int avg_nr_run = avg_nr_running(); + unsigned int nr_run; + + if (runnables_state == DISABLED) + return; + + for (nr_run = 1; nr_run < ARRAY_SIZE(nr_run_thresholds); nr_run++) { + unsigned int nr_threshold = nr_run_thresholds[nr_run - 1]; + if (nr_run_last <= nr_run) + nr_threshold += NR_FSHIFT / nr_run_hysteresis; + if (avg_nr_run <= (nr_threshold << (FSHIFT - NR_FSHIFT_EXP))) + break; + } + nr_run_last = nr_run; + + if ((nr_cpus > max_cpus || nr_run < nr_cpus) && nr_cpus >= min_cpus) { + runnables_state = DOWN; + } else if (nr_cpus < min_cpus || nr_run > nr_cpus) { + runnables_state = UP; + } else { + runnables_state = IDLE; + } +} + +static unsigned int get_lightest_loaded_cpu_n(void) +{ + unsigned long min_avg_runnables = ULONG_MAX; + unsigned int cpu = nr_cpu_ids; + int i; + + for_each_online_cpu(i) { + unsigned int nr_runnables = get_avg_nr_running(i); + + if (i > 0 && min_avg_runnables > nr_runnables) { + cpu = i; + min_avg_runnables = nr_runnables; + } + } + + return cpu; +} + +static void runnables_work_func(struct work_struct *work) +{ + bool up = false; + bool sample = false; + unsigned int cpu = nr_cpu_ids; + + mutex_lock(&runnables_work_lock); + + update_runnables_state(); + + switch (runnables_state) { + case DISABLED: + break; + case IDLE: + sample = true; + break; + case UP: + cpu = cpumask_next_zero(0, cpu_online_mask); + up = true; + sample = true; + break; + case DOWN: + cpu = get_lightest_loaded_cpu_n(); + sample = true; + break; + default: + pr_err("%s: invalid cpuquiet runnable governor state %d\n", + __func__, runnables_state); + break; + } + + if (sample) + queue_delayed_work(runnables_wq, &runnables_work, + msecs_to_jiffies(sample_rate)); + + if (cpu < nr_cpu_ids) { + if (up) + cpuquiet_wake_cpu(cpu); + else + cpuquiet_quiesence_cpu(cpu); + } + + mutex_unlock(&runnables_work_lock); +} + +CPQ_BASIC_ATTRIBUTE(sample_rate, 0644, uint); +CPQ_BASIC_ATTRIBUTE(nr_run_hysteresis, 0644, uint); + +static struct attribute *runnables_attributes[] = { + &sample_rate_attr.attr, + &nr_run_hysteresis_attr.attr, + NULL, +}; + +static const struct sysfs_ops runnables_sysfs_ops = { + .show = cpuquiet_auto_sysfs_show, + .store = cpuquiet_auto_sysfs_store, +}; + +static struct kobj_type ktype_runnables = { + .sysfs_ops = &runnables_sysfs_ops, + .default_attrs = runnables_attributes, +}; + +static int runnables_sysfs(void) +{ + int err; + + runnables_kobject = kzalloc(sizeof(*runnables_kobject), + GFP_KERNEL); + + if (!runnables_kobject) + return -ENOMEM; + + err = cpuquiet_kobject_init(runnables_kobject, &ktype_runnables, + "runnable_threads"); + + if (err) + kfree(runnables_kobject); + + return err; +} + +static void runnables_stop(void) +{ + runnables_state = DISABLED; + cancel_delayed_work_sync(&runnables_work); + destroy_workqueue(runnables_wq); + kobject_put(runnables_kobject); +} + +static int runnables_start(void) +{ + int err, i; + + err = runnables_sysfs(); + if (err) + return err; + + runnables_wq = alloc_workqueue("cpuquiet-runnables", + WQ_UNBOUND | WQ_RESCUER | WQ_FREEZABLE, 1); + if (!runnables_wq) + return -ENOMEM; + + INIT_DELAYED_WORK(&runnables_work, runnables_work_func); + + for(i = 0; i < ARRAY_SIZE(nr_run_thresholds); ++i) { + if (i < ARRAY_SIZE(default_thresholds)) + nr_run_thresholds[i] = default_thresholds[i]; + else if (i == (ARRAY_SIZE(nr_run_thresholds) - 1)) + nr_run_thresholds[i] = UINT_MAX; + else + nr_run_thresholds[i] = i + 1 + + NR_FSHIFT / default_threshold_level; + } + + runnables_state = IDLE; + runnables_work_func(NULL); + + return 0; +} + +struct cpuquiet_governor runnables_governor = { + .name = "runnable", + .start = runnables_start, + .stop = runnables_stop, + .owner = THIS_MODULE, +}; + +static int __init init_runnables(void) +{ + return cpuquiet_register_governor(&runnables_governor); +} + +static void __exit exit_runnables(void) +{ + cpuquiet_unregister_governor(&runnables_governor); +} + +MODULE_LICENSE("GPL"); +module_init(init_runnables); +module_exit(exit_runnables); diff --git a/include/linux/sched.h b/include/linux/sched.h index c8c3b64ee91f..76137cba8e63 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -139,6 +139,7 @@ extern int nr_processes(void); extern unsigned long nr_running(void); extern unsigned long nr_uninterruptible(void); extern unsigned long nr_iowait(void); +extern unsigned long get_avg_nr_running(unsigned int cpu); extern unsigned long avg_nr_running(void); extern unsigned long nr_iowait_cpu(int cpu); extern unsigned long this_cpu_load(void); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index eb15edd08a23..b8f4618407b2 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2188,6 +2188,18 @@ unsigned long avg_nr_running(void) return sum; } +unsigned long get_avg_nr_running(unsigned int cpu) +{ + struct rq *q; + + if (cpu >= nr_cpu_ids) + return 0; + + q = cpu_rq(cpu); + + return q->ave_nr_running; +} + /* * Global load-average calculations * -- cgit v1.2.3