diff options
Diffstat (limited to 'drivers/cpufreq/cpufreq_adaptive.c')
-rw-r--r-- | drivers/cpufreq/cpufreq_adaptive.c | 952 |
1 files changed, 0 insertions, 952 deletions
diff --git a/drivers/cpufreq/cpufreq_adaptive.c b/drivers/cpufreq/cpufreq_adaptive.c deleted file mode 100644 index ad7f7de..0000000 --- a/drivers/cpufreq/cpufreq_adaptive.c +++ /dev/null @@ -1,952 +0,0 @@ -/* - * drivers/cpufreq/cpufreq_adaptive.c - * - * Copyright (C) 2001 Russell King - * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>. - * Jun Nakajima <jun.nakajima@intel.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/init.h> -#include <linux/cpufreq.h> -#include <linux/cpu.h> -#include <linux/jiffies.h> -#include <linux/kernel_stat.h> -#include <linux/mutex.h> -#include <linux/hrtimer.h> -#include <linux/tick.h> -#include <linux/ktime.h> -#include <linux/sched.h> -#include <linux/kthread.h> - -#include <mach/ppmu.h> - -/* - * dbs is used in this file as a shortform for demandbased switching - * It helps to keep variable names smaller, simpler - */ - -#define DEF_FREQUENCY_DOWN_DIFFERENTIAL (10) -#define DEF_FREQUENCY_UP_THRESHOLD (80) -#define MICRO_FREQUENCY_DOWN_DIFFERENTIAL (3) -#define MICRO_FREQUENCY_UP_THRESHOLD (95) -#define MICRO_FREQUENCY_MIN_SAMPLE_RATE (10000) -#define MIN_FREQUENCY_UP_THRESHOLD (11) -#define MAX_FREQUENCY_UP_THRESHOLD (100) -#define MIN_ONDEMAND_THRESHOLD (4) -/* - * The polling frequency of this governor depends on the capability of - * the processor. Default polling frequency is 1000 times the transition - * latency of the processor. The governor will work on any processor with - * transition latency <= 10mS, using appropriate sampling - * rate. - * For CPUs with transition latency > 10mS (mostly drivers with CPUFREQ_ETERNAL) - * this governor will not work. - * All times here are in uS. - */ -#define MIN_SAMPLING_RATE_RATIO (2) - -static unsigned int min_sampling_rate; - -#define LATENCY_MULTIPLIER (1000) -#define MIN_LATENCY_MULTIPLIER (100) -#define TRANSITION_LATENCY_LIMIT (10 * 1000 * 1000) - -static void (*pm_idle_old)(void); -static void do_dbs_timer(struct work_struct *work); -static int cpufreq_governor_dbs(struct cpufreq_policy *policy, - unsigned int event); - -#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ADAPTIVE -static -#endif -struct cpufreq_governor cpufreq_gov_adaptive = { - .name = "adaptive", - .governor = cpufreq_governor_dbs, - .max_transition_latency = TRANSITION_LATENCY_LIMIT, - .owner = THIS_MODULE, -}; - -/* Sampling types */ -enum {DBS_NORMAL_SAMPLE, DBS_SUB_SAMPLE}; - -struct cpu_dbs_info_s { - cputime64_t prev_cpu_idle; - cputime64_t prev_cpu_iowait; - cputime64_t prev_cpu_wall; - cputime64_t prev_cpu_nice; - struct cpufreq_policy *cur_policy; - struct delayed_work work; - struct cpufreq_frequency_table *freq_table; - unsigned int freq_hi_jiffies; - int cpu; - unsigned int sample_type:1; - bool ondemand; - /* - * percpu mutex that serializes governor limit change with - * do_dbs_timer invocation. We do not want do_dbs_timer to run - * when user is changing the governor or limits. - */ - struct mutex timer_mutex; -}; -static DEFINE_PER_CPU(struct cpu_dbs_info_s, od_cpu_dbs_info); - -static unsigned int dbs_enable; /* number of CPUs using this policy */ - -/* - * dbs_mutex protects data in dbs_tuners_ins from concurrent changes on - * different CPUs. It protects dbs_enable in governor start/stop. - */ -static DEFINE_MUTEX(dbs_mutex); -static struct task_struct *up_task; -static struct workqueue_struct *down_wq; -static struct work_struct freq_scale_down_work; -static cpumask_t up_cpumask; -static spinlock_t up_cpumask_lock; -static cpumask_t down_cpumask; -static spinlock_t down_cpumask_lock; - -static DEFINE_PER_CPU(cputime64_t, idle_in_idle); -static DEFINE_PER_CPU(cputime64_t, idle_exit_wall); - -static struct timer_list cpu_timer; -static unsigned int target_freq; -static DEFINE_MUTEX(short_timer_mutex); - -/* Go to max speed when CPU load at or above this value. */ -#define DEFAULT_GO_MAXSPEED_LOAD 60 -static unsigned long go_maxspeed_load; - -#define DEFAULT_KEEP_MINSPEED_LOAD 30 -static unsigned long keep_minspeed_load; - -#define DEFAULT_STEPUP_LOAD 10 -static unsigned long step_up_load; - -static struct dbs_tuners { - unsigned int sampling_rate; - unsigned int up_threshold; - unsigned int down_differential; - unsigned int ignore_nice; - unsigned int io_is_busy; -} dbs_tuners_ins = { - .up_threshold = DEF_FREQUENCY_UP_THRESHOLD, - .down_differential = DEF_FREQUENCY_DOWN_DIFFERENTIAL, - .ignore_nice = 0, -}; - -static inline cputime64_t get_cpu_iowait_time(unsigned int cpu, cputime64_t *wall) -{ - u64 iowait_time = get_cpu_iowait_time_us(cpu, wall); - - if (iowait_time == -1ULL) - return 0; - - return iowait_time; -} - -static void adaptive_init_cpu(int cpu) -{ - struct cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu); - dbs_info->freq_table = cpufreq_frequency_get_table(cpu); -} - -/************************** sysfs interface ************************/ - -static ssize_t show_sampling_rate_max(struct kobject *kobj, - struct attribute *attr, char *buf) -{ - printk_once(KERN_INFO "CPUFREQ: adaptive sampling_rate_max " - "sysfs file is deprecated - used by: %s\n", current->comm); - return sprintf(buf, "%u\n", -1U); -} - -static ssize_t show_sampling_rate_min(struct kobject *kobj, - struct attribute *attr, char *buf) -{ - return sprintf(buf, "%u\n", min_sampling_rate); -} - -define_one_global_ro(sampling_rate_max); -define_one_global_ro(sampling_rate_min); - -/* cpufreq_adaptive Governor Tunables */ -#define show_one(file_name, object) \ -static ssize_t show_##file_name \ -(struct kobject *kobj, struct attribute *attr, char *buf) \ -{ \ - return sprintf(buf, "%u\n", dbs_tuners_ins.object); \ -} -show_one(sampling_rate, sampling_rate); -show_one(io_is_busy, io_is_busy); -show_one(up_threshold, up_threshold); -show_one(ignore_nice_load, ignore_nice); - -/*** delete after deprecation time ***/ - -#define DEPRECATION_MSG(file_name) \ - printk_once(KERN_INFO "CPUFREQ: Per core adaptive sysfs " \ - "interface is deprecated - " #file_name "\n"); - -#define show_one_old(file_name) \ -static ssize_t show_##file_name##_old \ -(struct cpufreq_policy *unused, char *buf) \ -{ \ - printk_once(KERN_INFO "CPUFREQ: Per core adaptive sysfs " \ - "interface is deprecated - " #file_name "\n"); \ - return show_##file_name(NULL, NULL, buf); \ -} - -/*** delete after deprecation time ***/ - -static ssize_t store_sampling_rate(struct kobject *a, struct attribute *b, - const char *buf, size_t count) -{ - unsigned int input; - int ret; - ret = sscanf(buf, "%u", &input); - if (ret != 1) - return -EINVAL; - - mutex_lock(&dbs_mutex); - dbs_tuners_ins.sampling_rate = max(input, min_sampling_rate); - mutex_unlock(&dbs_mutex); - - return count; -} - -static ssize_t store_io_is_busy(struct kobject *a, struct attribute *b, - const char *buf, size_t count) -{ - unsigned int input; - int ret; - - ret = sscanf(buf, "%u", &input); - if (ret != 1) - return -EINVAL; - - mutex_lock(&dbs_mutex); - dbs_tuners_ins.io_is_busy = !!input; - mutex_unlock(&dbs_mutex); - - return count; -} - -static ssize_t store_up_threshold(struct kobject *a, struct attribute *b, - const char *buf, size_t count) -{ - unsigned int input; - int ret; - ret = sscanf(buf, "%u", &input); - - if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD || - input < MIN_FREQUENCY_UP_THRESHOLD) { - return -EINVAL; - } - - mutex_lock(&dbs_mutex); - dbs_tuners_ins.up_threshold = input; - mutex_unlock(&dbs_mutex); - - return count; -} - -static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b, - const char *buf, size_t count) -{ - unsigned int input; - int ret; - - unsigned int j; - - ret = sscanf(buf, "%u", &input); - if (ret != 1) - return -EINVAL; - - if (input > 1) - input = 1; - - mutex_lock(&dbs_mutex); - if (input == dbs_tuners_ins.ignore_nice) { /* nothing to do */ - mutex_unlock(&dbs_mutex); - return count; - } - dbs_tuners_ins.ignore_nice = input; - - /* we need to re-evaluate prev_cpu_idle */ - for_each_online_cpu(j) { - struct cpu_dbs_info_s *dbs_info; - dbs_info = &per_cpu(od_cpu_dbs_info, j); - dbs_info->prev_cpu_idle = get_cpu_idle_time_us(j, - &dbs_info->prev_cpu_wall); - if (dbs_tuners_ins.ignore_nice) - dbs_info->prev_cpu_nice = kstat_cpu(j).cpustat.nice; - - } - mutex_unlock(&dbs_mutex); - - return count; -} - -define_one_global_rw(sampling_rate); -define_one_global_rw(io_is_busy); -define_one_global_rw(up_threshold); -define_one_global_rw(ignore_nice_load); - -static struct attribute *dbs_attributes[] = { - &sampling_rate_max.attr, - &sampling_rate_min.attr, - &sampling_rate.attr, - &up_threshold.attr, - &ignore_nice_load.attr, - &io_is_busy.attr, - NULL -}; - -static struct attribute_group dbs_attr_group = { - .attrs = dbs_attributes, - .name = "adaptive", -}; - -/*** delete after deprecation time ***/ - -#define write_one_old(file_name) \ -static ssize_t store_##file_name##_old \ -(struct cpufreq_policy *unused, const char *buf, size_t count) \ -{ \ - printk_once(KERN_INFO "CPUFREQ: Per core adaptive sysfs " \ - "interface is deprecated - " #file_name "\n"); \ - return store_##file_name(NULL, NULL, buf, count); \ -} - -static void cpufreq_adaptive_timer(unsigned long data) -{ - cputime64_t cur_idle; - cputime64_t cur_wall; - unsigned int delta_idle; - unsigned int delta_time; - int short_load; - unsigned int new_freq; - unsigned long flags; - struct cpu_dbs_info_s *this_dbs_info; - struct cpufreq_policy *policy; - unsigned int j; - unsigned int index; - unsigned int max_load = 0; - - this_dbs_info = &per_cpu(od_cpu_dbs_info, 0); - - policy = this_dbs_info->cur_policy; - - for_each_online_cpu(j) { - cur_idle = get_cpu_idle_time_us(j, &cur_wall); - - delta_idle = (unsigned int) cputime64_sub(cur_idle, - per_cpu(idle_in_idle, j)); - delta_time = (unsigned int) cputime64_sub(cur_wall, - per_cpu(idle_exit_wall, j)); - - /* - * If timer ran less than 1ms after short-term sample started, retry. - */ - if (delta_time < 1000) - goto do_nothing; - - if (delta_idle > delta_time) - short_load = 0; - else - short_load = 100 * (delta_time - delta_idle) / delta_time; - - if (short_load > max_load) - max_load = short_load; - } - - if (this_dbs_info->ondemand) - goto do_nothing; - - if (max_load >= go_maxspeed_load) - new_freq = policy->max; - else - new_freq = policy->max * max_load / 100; - - if ((max_load <= keep_minspeed_load) && - (policy->cur == policy->min)) - new_freq = policy->cur; - - if (cpufreq_frequency_table_target(policy, this_dbs_info->freq_table, - new_freq, CPUFREQ_RELATION_L, - &index)) { - goto do_nothing; - } - - new_freq = this_dbs_info->freq_table[index].frequency; - - target_freq = new_freq; - - if (new_freq < this_dbs_info->cur_policy->cur) { - spin_lock_irqsave(&down_cpumask_lock, flags); - cpumask_set_cpu(0, &down_cpumask); - spin_unlock_irqrestore(&down_cpumask_lock, flags); - queue_work(down_wq, &freq_scale_down_work); - } else { - spin_lock_irqsave(&up_cpumask_lock, flags); - cpumask_set_cpu(0, &up_cpumask); - spin_unlock_irqrestore(&up_cpumask_lock, flags); - wake_up_process(up_task); - } - - return; - -do_nothing: - for_each_online_cpu(j) { - per_cpu(idle_in_idle, j) = - get_cpu_idle_time_us(j, - &per_cpu(idle_exit_wall, j)); - } - mod_timer(&cpu_timer, jiffies + 2); - schedule_delayed_work_on(0, &this_dbs_info->work, 10); - - if (mutex_is_locked(&short_timer_mutex)) - mutex_unlock(&short_timer_mutex); - return; -} - -/*** delete after deprecation time ***/ - -/************************** sysfs end ************************/ - -static void dbs_freq_increase(struct cpufreq_policy *p, unsigned int freq) -{ -#ifndef CONFIG_ARCH_EXYNOS4 - if (p->cur == p->max) - return; -#endif - __cpufreq_driver_target(p, freq, CPUFREQ_RELATION_H); -} - -static void dbs_check_cpu(struct cpu_dbs_info_s *this_dbs_info) -{ - unsigned int max_load_freq; - - struct cpufreq_policy *policy; - unsigned int j; - - unsigned int index, new_freq; - unsigned int longterm_load = 0; - - policy = this_dbs_info->cur_policy; - - /* - * Every sampling_rate, we check, if current idle time is less - * than 20% (default), then we try to increase frequency - * Every sampling_rate, we look for a the lowest - * frequency which can sustain the load while keeping idle time over - * 30%. If such a frequency exist, we try to decrease to this frequency. - * - * Any frequency increase takes it to the maximum frequency. - * Frequency reduction happens at minimum steps of - * 5% (default) of current frequency - */ - - /* Get Absolute Load - in terms of freq */ - max_load_freq = 0; - - for_each_cpu(j, policy->cpus) { - struct cpu_dbs_info_s *j_dbs_info; - cputime64_t cur_wall_time, cur_idle_time, cur_iowait_time; - unsigned int idle_time, wall_time, iowait_time; - unsigned int load, load_freq; - int freq_avg; - - j_dbs_info = &per_cpu(od_cpu_dbs_info, j); - - cur_idle_time = get_cpu_idle_time_us(j, &cur_wall_time); - cur_iowait_time = get_cpu_iowait_time(j, &cur_wall_time); - - wall_time = (unsigned int) cputime64_sub(cur_wall_time, - j_dbs_info->prev_cpu_wall); - j_dbs_info->prev_cpu_wall = cur_wall_time; - - idle_time = (unsigned int) cputime64_sub(cur_idle_time, - j_dbs_info->prev_cpu_idle); - j_dbs_info->prev_cpu_idle = cur_idle_time; - - iowait_time = (unsigned int) cputime64_sub(cur_iowait_time, - j_dbs_info->prev_cpu_iowait); - j_dbs_info->prev_cpu_iowait = cur_iowait_time; - - if (dbs_tuners_ins.ignore_nice) { - cputime64_t cur_nice; - unsigned long cur_nice_jiffies; - - cur_nice = cputime64_sub(kstat_cpu(j).cpustat.nice, - j_dbs_info->prev_cpu_nice); - /* - * Assumption: nice time between sampling periods will - * be less than 2^32 jiffies for 32 bit sys - */ - cur_nice_jiffies = (unsigned long) - cputime64_to_jiffies64(cur_nice); - - j_dbs_info->prev_cpu_nice = kstat_cpu(j).cpustat.nice; - idle_time += jiffies_to_usecs(cur_nice_jiffies); - } - - /* - * For the purpose of adaptive, waiting for disk IO is an - * indication that you're performance critical, and not that - * the system is actually idle. So subtract the iowait time - * from the cpu idle time. - */ - - if (dbs_tuners_ins.io_is_busy && idle_time >= iowait_time) - idle_time -= iowait_time; - - if (unlikely(!wall_time || wall_time < idle_time)) - continue; - - load = 100 * (wall_time - idle_time) / wall_time; - - if (load > longterm_load) - longterm_load = load; - - freq_avg = __cpufreq_driver_getavg(policy, j); - if (freq_avg <= 0) - freq_avg = policy->cur; - - load_freq = load * freq_avg; - - if (load_freq > max_load_freq) - max_load_freq = load_freq; - } - - if (longterm_load >= MIN_ONDEMAND_THRESHOLD) - this_dbs_info->ondemand = true; - else - this_dbs_info->ondemand = false; - - /* Check for frequency increase */ - if (max_load_freq > (dbs_tuners_ins.up_threshold * policy->cur)) { - cpufreq_frequency_table_target(policy, - this_dbs_info->freq_table, - (policy->cur + step_up_load), - CPUFREQ_RELATION_L, &index); - - new_freq = this_dbs_info->freq_table[index].frequency; - dbs_freq_increase(policy, new_freq); - return; - } - - /* Check for frequency decrease */ - /* if we cannot reduce the frequency anymore, break out early */ -#ifndef CONFIG_ARCH_EXYNOS4 - if (policy->cur == policy->min) - return; -#endif - /* - * The optimal frequency is the frequency that is the lowest that - * can support the current CPU usage without triggering the up - * policy. To be safe, we focus 10 points under the threshold. - */ - if (max_load_freq < - (dbs_tuners_ins.up_threshold - dbs_tuners_ins.down_differential) * - policy->cur) { - unsigned int freq_next; - freq_next = max_load_freq / - (dbs_tuners_ins.up_threshold - - dbs_tuners_ins.down_differential); - - if (freq_next < policy->min) - freq_next = policy->min; - - __cpufreq_driver_target(policy, freq_next, - CPUFREQ_RELATION_L); - } -} - -static void do_dbs_timer(struct work_struct *work) -{ - struct cpu_dbs_info_s *dbs_info = - container_of(work, struct cpu_dbs_info_s, work.work); - unsigned int cpu = dbs_info->cpu; - - int delay; - - mutex_lock(&dbs_info->timer_mutex); - - /* Common NORMAL_SAMPLE setup */ - dbs_info->sample_type = DBS_NORMAL_SAMPLE; - dbs_check_cpu(dbs_info); - - /* We want all CPUs to do sampling nearly on - * same jiffy - */ - delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate); - - schedule_delayed_work_on(cpu, &dbs_info->work, delay); - - mutex_unlock(&dbs_info->timer_mutex); -} - -static inline void dbs_timer_init(struct cpu_dbs_info_s *dbs_info) -{ - /* We want all CPUs to do sampling nearly on same jiffy */ - int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate); - - dbs_info->sample_type = DBS_NORMAL_SAMPLE; - INIT_DELAYED_WORK_DEFERRABLE(&dbs_info->work, do_dbs_timer); - schedule_delayed_work_on(dbs_info->cpu, &dbs_info->work, delay); -} - -static inline void dbs_timer_exit(struct cpu_dbs_info_s *dbs_info) -{ - cancel_delayed_work_sync(&dbs_info->work); -} - -/* - * Not all CPUs want IO time to be accounted as busy; this dependson how - * efficient idling at a higher frequency/voltage is. - * Pavel Machek says this is not so for various generations of AMD and old - * Intel systems. - * Mike Chan (androidlcom) calis this is also not true for ARM. - * Because of this, whitelist specific known (series) of CPUs by default, and - * leave all others up to the user. - */ -static int should_io_be_busy(void) -{ -#if defined(CONFIG_X86) - /* - * For Intel, Core 2 (model 15) andl later have an efficient idle. - */ - if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && - boot_cpu_data.x86 == 6 && - boot_cpu_data.x86_model >= 15) - return 1; -#endif - return 0; -} - -static void cpufreq_adaptive_idle(void) -{ - int i; - struct cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, 0); - struct cpufreq_policy *policy; - - policy = dbs_info->cur_policy; - - pm_idle_old(); - - if ((policy->cur == policy->min) || - (policy->cur == policy->max)) { - - if (timer_pending(&cpu_timer)) - return; - - if (mutex_trylock(&short_timer_mutex)) { - for_each_online_cpu(i) { - per_cpu(idle_in_idle, i) = - get_cpu_idle_time_us(i, - &per_cpu(idle_exit_wall, i)); - } - - mod_timer(&cpu_timer, jiffies + 2); - cancel_delayed_work(&dbs_info->work); - } - } else { - if (timer_pending(&cpu_timer)) - del_timer(&cpu_timer); - - } -} - -static int cpufreq_governor_dbs(struct cpufreq_policy *policy, - unsigned int event) -{ - unsigned int cpu = policy->cpu; - struct cpu_dbs_info_s *this_dbs_info; - unsigned int j; - int rc; - - this_dbs_info = &per_cpu(od_cpu_dbs_info, cpu); - - switch (event) { - case CPUFREQ_GOV_START: - if ((!cpu_online(cpu)) || (!policy->cur)) - return -EINVAL; - - mutex_lock(&dbs_mutex); - - rc = sysfs_create_group(&policy->kobj, &dbs_attr_group); - if (rc) { - mutex_unlock(&dbs_mutex); - return rc; - } - - dbs_enable++; - for_each_cpu(j, policy->cpus) { - struct cpu_dbs_info_s *j_dbs_info; - j_dbs_info = &per_cpu(od_cpu_dbs_info, j); - j_dbs_info->cur_policy = policy; - - j_dbs_info->prev_cpu_idle = get_cpu_idle_time_us(j, - &j_dbs_info->prev_cpu_wall); - if (dbs_tuners_ins.ignore_nice) { - j_dbs_info->prev_cpu_nice = - kstat_cpu(j).cpustat.nice; - } - } - this_dbs_info->cpu = cpu; - adaptive_init_cpu(cpu); - - /* - * Start the timerschedule work, when this governor - * is used for first time - */ - if (dbs_enable == 1) { - unsigned int latency; - - rc = sysfs_create_group(cpufreq_global_kobject, - &dbs_attr_group); - if (rc) { - mutex_unlock(&dbs_mutex); - return rc; - } - - /* policy latency is in nS. Convert it to uS first */ - latency = policy->cpuinfo.transition_latency / 1000; - if (latency == 0) - latency = 1; - /* Bring kernel and HW constraints together */ - min_sampling_rate = max(min_sampling_rate, - MIN_LATENCY_MULTIPLIER * latency); - dbs_tuners_ins.sampling_rate = - max(min_sampling_rate, - latency * LATENCY_MULTIPLIER); - dbs_tuners_ins.io_is_busy = should_io_be_busy(); - } - mutex_unlock(&dbs_mutex); - - mutex_init(&this_dbs_info->timer_mutex); - dbs_timer_init(this_dbs_info); - - pm_idle_old = pm_idle; - pm_idle = cpufreq_adaptive_idle; - break; - - case CPUFREQ_GOV_STOP: - dbs_timer_exit(this_dbs_info); - - mutex_lock(&dbs_mutex); - sysfs_remove_group(&policy->kobj, &dbs_attr_group); - mutex_destroy(&this_dbs_info->timer_mutex); - dbs_enable--; - mutex_unlock(&dbs_mutex); - if (!dbs_enable) - sysfs_remove_group(cpufreq_global_kobject, - &dbs_attr_group); - - pm_idle = pm_idle_old; - break; - - case CPUFREQ_GOV_LIMITS: - mutex_lock(&this_dbs_info->timer_mutex); - if (policy->max < this_dbs_info->cur_policy->cur) - __cpufreq_driver_target(this_dbs_info->cur_policy, - policy->max, CPUFREQ_RELATION_H); - else if (policy->min > this_dbs_info->cur_policy->cur) - __cpufreq_driver_target(this_dbs_info->cur_policy, - policy->min, CPUFREQ_RELATION_L); - mutex_unlock(&this_dbs_info->timer_mutex); - break; - } - return 0; -} - -static inline void cpufreq_adaptive_update_time(void) -{ - struct cpu_dbs_info_s *this_dbs_info; - struct cpufreq_policy *policy; - int j; - - this_dbs_info = &per_cpu(od_cpu_dbs_info, 0); - policy = this_dbs_info->cur_policy; - - for_each_cpu(j, policy->cpus) { - struct cpu_dbs_info_s *j_dbs_info; - cputime64_t cur_wall_time, cur_idle_time, cur_iowait_time; - - j_dbs_info = &per_cpu(od_cpu_dbs_info, j); - - cur_idle_time = get_cpu_idle_time_us(j, &cur_wall_time); - cur_iowait_time = get_cpu_iowait_time(j, &cur_wall_time); - - j_dbs_info->prev_cpu_wall = cur_wall_time; - - j_dbs_info->prev_cpu_idle = cur_idle_time; - - j_dbs_info->prev_cpu_iowait = cur_iowait_time; - - if (dbs_tuners_ins.ignore_nice) - j_dbs_info->prev_cpu_nice = kstat_cpu(j).cpustat.nice; - - } - -} - -static int cpufreq_adaptive_up_task(void *data) -{ - unsigned long flags; - struct cpu_dbs_info_s *this_dbs_info; - struct cpufreq_policy *policy; - int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate); - - this_dbs_info = &per_cpu(od_cpu_dbs_info, 0); - policy = this_dbs_info->cur_policy; - - while (1) { - set_current_state(TASK_INTERRUPTIBLE); - spin_lock_irqsave(&up_cpumask_lock, flags); - - if (cpumask_empty(&up_cpumask)) { - spin_unlock_irqrestore(&up_cpumask_lock, flags); - schedule(); - - if (kthread_should_stop()) - break; - - spin_lock_irqsave(&up_cpumask_lock, flags); - } - - set_current_state(TASK_RUNNING); - - cpumask_clear(&up_cpumask); - spin_unlock_irqrestore(&up_cpumask_lock, flags); - - __cpufreq_driver_target(this_dbs_info->cur_policy, - target_freq, - CPUFREQ_RELATION_H); - if (policy->cur != policy->max) { - mutex_lock(&this_dbs_info->timer_mutex); - - schedule_delayed_work_on(0, &this_dbs_info->work, delay); - mutex_unlock(&this_dbs_info->timer_mutex); - cpufreq_adaptive_update_time(); - } - if (mutex_is_locked(&short_timer_mutex)) - mutex_unlock(&short_timer_mutex); - } - - return 0; -} - -static void cpufreq_adaptive_freq_down(struct work_struct *work) -{ - unsigned long flags; - struct cpu_dbs_info_s *this_dbs_info; - struct cpufreq_policy *policy; - int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate); - - spin_lock_irqsave(&down_cpumask_lock, flags); - cpumask_clear(&down_cpumask); - spin_unlock_irqrestore(&down_cpumask_lock, flags); - - this_dbs_info = &per_cpu(od_cpu_dbs_info, 0); - policy = this_dbs_info->cur_policy; - - __cpufreq_driver_target(this_dbs_info->cur_policy, - target_freq, - CPUFREQ_RELATION_H); - - if (policy->cur != policy->min) { - mutex_lock(&this_dbs_info->timer_mutex); - - schedule_delayed_work_on(0, &this_dbs_info->work, delay); - mutex_unlock(&this_dbs_info->timer_mutex); - cpufreq_adaptive_update_time(); - } - - if (mutex_is_locked(&short_timer_mutex)) - mutex_unlock(&short_timer_mutex); -} - -static int __init cpufreq_gov_dbs_init(void) -{ - cputime64_t wall; - u64 idle_time; - int cpu = get_cpu(); - - struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; - go_maxspeed_load = DEFAULT_GO_MAXSPEED_LOAD; - keep_minspeed_load = DEFAULT_KEEP_MINSPEED_LOAD; - step_up_load = DEFAULT_STEPUP_LOAD; - - idle_time = get_cpu_idle_time_us(cpu, &wall); - put_cpu(); - if (idle_time != -1ULL) { - /* Idle micro accounting is supported. Use finer thresholds */ - dbs_tuners_ins.up_threshold = MICRO_FREQUENCY_UP_THRESHOLD; - dbs_tuners_ins.down_differential = - MICRO_FREQUENCY_DOWN_DIFFERENTIAL; - /* - * In no_hz/micro accounting case we set the minimum frequency - * not depending on HZ, but fixed (very low). The deferred - * timer might skip some samples if idle/sleeping as needed. - */ - min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE; - } else { - /* For correct statistics, we need 10 ticks for each measure */ - min_sampling_rate = - MIN_SAMPLING_RATE_RATIO * jiffies_to_usecs(10); - } - - init_timer(&cpu_timer); - cpu_timer.function = cpufreq_adaptive_timer; - - up_task = kthread_create(cpufreq_adaptive_up_task, NULL, - "kadaptiveup"); - - if (IS_ERR(up_task)) - return PTR_ERR(up_task); - - sched_setscheduler_nocheck(up_task, SCHED_FIFO, ¶m); - get_task_struct(up_task); - - /* No rescuer thread, bind to CPU queuing the work for possibly - warm cache (probably doesn't matter much). */ - down_wq = alloc_workqueue("kadaptive_down", 0, 1); - - if (!down_wq) - goto err_freeuptask; - - INIT_WORK(&freq_scale_down_work, cpufreq_adaptive_freq_down); - - - return cpufreq_register_governor(&cpufreq_gov_adaptive); -err_freeuptask: - put_task_struct(up_task); - return -ENOMEM; -} - -static void __exit cpufreq_gov_dbs_exit(void) -{ - cpufreq_unregister_governor(&cpufreq_gov_adaptive); -} - - -MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>"); -MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>"); -MODULE_DESCRIPTION("'cpufreq_adaptive' - A dynamic cpufreq governor for " - "Low Latency Frequency Transition capable processors"); -MODULE_LICENSE("GPL"); - -#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ADAPTIVE -fs_initcall(cpufreq_gov_dbs_init); -#else -module_init(cpufreq_gov_dbs_init); -#endif -module_exit(cpufreq_gov_dbs_exit); |