From 5b465b045af3a649a89b8a5c5bfdece20ffc0345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Sat, 17 Sep 2016 02:25:44 -0300 Subject: Linux-libre 4.7.4-gnu --- kernel/sched/bfs.c | 265 ++++++++--------------------------------------------- 1 file changed, 37 insertions(+), 228 deletions(-) (limited to 'kernel/sched/bfs.c') diff --git a/kernel/sched/bfs.c b/kernel/sched/bfs.c index 4168a5527..60b9ab785 100644 --- a/kernel/sched/bfs.c +++ b/kernel/sched/bfs.c @@ -137,7 +137,7 @@ void print_scheduler_version(void) { - printk(KERN_INFO "BFS CPU scheduler v0.490 by Con Kolivas.\n"); + printk(KERN_INFO "BFS CPU scheduler v0.497 by Con Kolivas.\n"); } /* @@ -204,7 +204,7 @@ struct global_rq { int iso_ticks; bool iso_refractory; - skiplist_node *node; + skiplist_node node; skiplist *sl; }; @@ -546,7 +546,7 @@ static inline bool deadline_after(u64 deadline, u64 time) */ static inline bool task_queued(struct task_struct *p) { - return p->node; + return !skiplist_node_empty(&p->node); } /* @@ -557,8 +557,7 @@ static inline bool task_queued(struct task_struct *p) */ static void dequeue_task(struct task_struct *p) { - skiplist_delnode(grq.node, grq.sl, p->node); - p->node = NULL; + skiplist_delete(grq.sl, &p->node); sched_info_dequeued(task_rq(p), p); } @@ -622,7 +621,7 @@ static void enqueue_task(struct task_struct *p, struct rq *rq) * so mask out ~microseconds as the random seed for skiplist insertion. */ randseed = (grq.niffies >> 10) & 0xFFFFFFFF; - p->node = skiplist_insert(grq.node, grq.sl, sl_id, p, randseed); + skiplist_insert(grq.sl, &p->node, sl_id, p, randseed); sched_info_queued(rq, p); } @@ -693,7 +692,7 @@ static int best_smt_bias(struct rq *this_rq) if (rq_idle(rq)) continue; - if (!rq->online) + if (unlikely(!rq->online)) continue; if (!rq->rq_mm) continue; @@ -818,10 +817,7 @@ static bool suitable_idle_cpus(struct task_struct *p) #define CPUIDLE_CACHE_BUSY (4) #define CPUIDLE_DIFF_CPU (8) #define CPUIDLE_THREAD_BUSY (16) -#define CPUIDLE_THROTTLED (32) -#define CPUIDLE_DIFF_NODE (64) - -static inline bool scaling_rq(struct rq *rq); +#define CPUIDLE_DIFF_NODE (32) /* * The best idle CPU is chosen according to the CPUIDLE ranking above where the @@ -840,9 +836,9 @@ static inline bool scaling_rq(struct rq *rq); */ static int best_mask_cpu(int best_cpu, struct rq *rq, cpumask_t *tmpmask) { - int best_ranking = CPUIDLE_DIFF_NODE | CPUIDLE_THROTTLED | - CPUIDLE_THREAD_BUSY | CPUIDLE_DIFF_CPU | CPUIDLE_CACHE_BUSY | - CPUIDLE_DIFF_CORE | CPUIDLE_DIFF_THREAD; + int best_ranking = CPUIDLE_DIFF_NODE | CPUIDLE_THREAD_BUSY | + CPUIDLE_DIFF_CPU | CPUIDLE_CACHE_BUSY | CPUIDLE_DIFF_CORE | + CPUIDLE_DIFF_THREAD; int cpu_tmp; if (cpumask_test_cpu(best_cpu, tmpmask)) @@ -875,9 +871,6 @@ static int best_mask_cpu(int best_cpu, struct rq *rq, cpumask_t *tmpmask) if (!(tmp_rq->siblings_idle(tmp_rq))) ranking |= CPUIDLE_THREAD_BUSY; #endif - if (scaling_rq(tmp_rq)) - ranking |= CPUIDLE_THROTTLED; - if (ranking < best_ranking) { best_cpu = cpu_tmp; best_ranking = ranking; @@ -887,12 +880,6 @@ out: return best_cpu; } -static void resched_best_mask(int best_cpu, struct rq *rq, cpumask_t *tmpmask) -{ - best_cpu = best_mask_cpu(best_cpu, rq, tmpmask); - resched_curr(cpu_rq(best_cpu)); -} - bool cpus_share_cache(int this_cpu, int that_cpu) { struct rq *this_rq = cpu_rq(this_cpu); @@ -920,25 +907,6 @@ static inline void resched_suitable_idle(struct task_struct *p) if (suitable_idle_cpus(p)) resched_best_idle(p); } -/* - * Flags to tell us whether this CPU is running a CPU frequency governor that - * has slowed its speed or not. No locking required as the very rare wrongly - * read value would be harmless. - */ -void cpu_scaling(int cpu) -{ - cpu_rq(cpu)->scaling = true; -} - -void cpu_nonscaling(int cpu) -{ - cpu_rq(cpu)->scaling = false; -} - -static inline bool scaling_rq(struct rq *rq) -{ - return rq->scaling; -} static inline int locality_diff(int cpu, struct rq *rq) { @@ -962,30 +930,11 @@ static inline void resched_suitable_idle(struct task_struct *p) { } -void cpu_scaling(int __unused) -{ -} - -void cpu_nonscaling(int __unused) -{ -} - -/* - * Although CPUs can scale in UP, there is nowhere else for tasks to go so this - * always returns 0. - */ -static inline bool scaling_rq(struct rq *rq) -{ - return false; -} - static inline int locality_diff(int cpu, struct rq *rq) { return 0; } #endif /* CONFIG_SMP */ -EXPORT_SYMBOL_GPL(cpu_scaling); -EXPORT_SYMBOL_GPL(cpu_nonscaling); static inline int normal_prio(struct task_struct *p) { @@ -1018,8 +967,9 @@ static int effective_prio(struct task_struct *p) } /* - * Update the load average for feeding into cpu frequency governors. Use a rolling - * average with ~ time constant of 32ms + * Update the load average for feeding into cpu frequency governors. Use a + * rough estimate of a rolling average with ~ time constant of 32ms. + * 80/128 ~ 0.63. * 80 / 32768 / 128 == * 5 / 262144 */ static void update_load_avg(struct rq *rq) { @@ -1028,10 +978,10 @@ static void update_load_avg(struct rq *rq) unsigned long us_interval = (rq->clock - rq->load_update) >> 10; long load; - load = rq->load_avg - (rq->load_avg * us_interval * 80 / 32768 / 128); + load = rq->load_avg - (rq->load_avg * us_interval * 5 / 262144); if (unlikely(load < 0)) load = 0; - load += rq->soft_affined * rq_load_avg(rq) * us_interval * 80 / 32768 / 128; + load += rq->soft_affined * rq_load_avg(rq) * us_interval * 5 / 262144; rq->load_avg = load; } rq->load_update = rq->clock; @@ -1067,8 +1017,6 @@ static void activate_task(struct task_struct *p, struct rq *rq) cpufreq_trigger(grq.niffies, rq->load_avg); } -static inline void clear_sticky(struct task_struct *p); - /* * deactivate_task - If it's running, it's not on the grq and we can just * decrement the nr_running. Enter with grq locked. @@ -1080,7 +1028,6 @@ static inline void deactivate_task(struct task_struct *p, struct rq *rq) rq->soft_affined--; p->on_rq = 0; grq.nr_running--; - clear_sticky(p); update_load_avg(rq); cpufreq_trigger(grq.niffies, rq->load_avg); } @@ -1126,83 +1073,7 @@ void set_task_cpu(struct task_struct *p, unsigned int cpu) } task_thread_info(p)->cpu = cpu; } - -static inline void clear_sticky(struct task_struct *p) -{ - p->sticky = false; -} - -static inline bool task_sticky(struct task_struct *p) -{ - return p->sticky; -} - -/* Reschedule the best idle CPU that is not this one. */ -static void -resched_closest_idle(struct rq *rq, int cpu, struct task_struct *p) -{ - cpumask_t tmpmask; - - cpumask_and(&tmpmask, &p->cpus_allowed, &grq.cpu_idle_map); - cpumask_clear_cpu(cpu, &tmpmask); - if (cpumask_empty(&tmpmask)) - return; - resched_best_mask(cpu, rq, &tmpmask); -} - -/* - * We set the sticky flag on a task that is descheduled involuntarily meaning - * it is awaiting further CPU time. If the last sticky task is still sticky - * but unlucky enough to not be the next task scheduled, we unstick it and try - * to find it an idle CPU. Realtime tasks do not stick to minimise their - * latency at all times. - */ -static inline void -swap_sticky(struct rq *rq, int cpu, struct task_struct *p) -{ - if (rq->sticky_task) { - if (rq->sticky_task == p) { - p->sticky = true; - return; - } - if (task_sticky(rq->sticky_task)) { - clear_sticky(rq->sticky_task); - resched_closest_idle(rq, cpu, rq->sticky_task); - } - } - if (!rt_task(p)) { - p->sticky = true; - rq->sticky_task = p; - } else { - resched_closest_idle(rq, cpu, p); - rq->sticky_task = NULL; - } -} - -static inline void unstick_task(struct rq *rq, struct task_struct *p) -{ - rq->sticky_task = NULL; - clear_sticky(p); -} -#else -static inline void clear_sticky(struct task_struct *p) -{ -} - -static inline bool task_sticky(struct task_struct *p) -{ - return false; -} - -static inline void -swap_sticky(struct rq *rq, int cpu, struct task_struct *p) -{ -} - -static inline void unstick_task(struct rq *rq, struct task_struct *p) -{ -} -#endif +#endif /* CONFIG_SMP */ /* * Move a task off the global queue and take it to a cpu for it will @@ -1212,7 +1083,6 @@ static inline void take_task(int cpu, struct task_struct *p) { set_task_cpu(p, cpu); dequeue_task(p); - clear_sticky(p); dec_qnr(); } @@ -1458,17 +1328,10 @@ static inline bool needs_other_cpu(struct task_struct *p, int cpu) static void try_preempt(struct task_struct *p, struct rq *this_rq) { int cpu, pcpu, highest_prio, highest_cpu; - struct rq *highest_prio_rq = NULL; + struct rq *highest_prio_rq; u64 latest_deadline; cpumask_t tmp; - /* - * We clear the sticky flag here because for a task to have called - * try_preempt with the sticky flag enabled means some complicated - * re-scheduling has occurred and we should ignore the sticky flag. - */ - clear_sticky(p); - if (suitable_idle_cpus(p) && resched_best_idle(p)) return; @@ -1476,10 +1339,7 @@ static void try_preempt(struct task_struct *p, struct rq *this_rq) if (p->policy == SCHED_IDLEPRIO) return; - if (likely(online_cpus(p))) - cpumask_and(&tmp, &cpu_online_map, &p->cpus_allowed); - else - return; + cpumask_and(&tmp, &cpu_online_map, &p->cpus_allowed); /* See if this task can preempt the task on the current CPU first. */ pcpu = cpu_of(this_rq); @@ -1492,6 +1352,7 @@ static void try_preempt(struct task_struct *p, struct rq *this_rq) } highest_prio = latest_deadline = 0; + highest_prio_rq = NULL; /* Now look for the CPU with the latest deadline */ for_each_cpu(cpu, &tmp) { @@ -1522,13 +1383,10 @@ static void try_preempt(struct task_struct *p, struct rq *this_rq) return; if (can_preempt(p, highest_prio, latest_deadline)) { /* - * If we have decided this task should preempt this CPU, - * set the task's CPU to match so there is no discrepancy - * in earliest_deadline_task which biases away tasks with - * a different CPU set. This means waking tasks are - * treated differently to rescheduling tasks in - * interactive mode. - */ + * If we have decided this task should preempt this CPU, + * set the task's CPU to match thereby speeding up matching + * this task in earliest_deadline_task. + */ set_task_cpu(p, highest_cpu); resched_curr(highest_prio_rq); } @@ -1803,6 +1661,7 @@ int sched_fork(unsigned long __maybe_unused clone_flags, struct task_struct *p) p->sched_time = p->stime_pc = p->utime_pc = 0; + skiplist_node_init(&p->node); /* * Revert to default priority/policy on fork if requested. @@ -1825,13 +1684,11 @@ int sched_fork(unsigned long __maybe_unused clone_flags, struct task_struct *p) p->sched_reset_on_fork = 0; } - p->node = NULL; #ifdef CONFIG_SCHED_INFO if (unlikely(sched_info_on())) memset(&p->sched_info, 0, sizeof(p->sched_info)); #endif p->on_cpu = false; - clear_sticky(p); init_task_preempt_count(p); return 0; } @@ -3394,10 +3251,10 @@ static inline struct task_struct *earliest_deadline_task(struct rq *rq, int cpu, struct task_struct *idle) { struct task_struct *edt = idle; - skiplist_node *node = grq.node; + skiplist_node *node = &grq.node; u64 earliest_deadline = ~0ULL; - while ((node = node->next[0]) != grq.node) { + while ((node = node->next[0]) != &grq.node) { struct task_struct *p = node->value; int tcpu; @@ -3409,11 +3266,8 @@ task_struct *earliest_deadline_task(struct rq *rq, int cpu, struct task_struct * continue; if (!sched_interactive && (tcpu = task_cpu(p)) != cpu) { - u64 dl; + u64 dl = p->deadline << locality_diff(tcpu, rq); - if (task_sticky(p) && scaling_rq(rq)) - continue; - dl = p->deadline << locality_diff(tcpu, rq); if (unlikely(!deadline_before(dl, earliest_deadline))) continue; earliest_deadline = dl; @@ -3531,7 +3385,7 @@ static void check_smt_siblings(struct rq *this_rq) rq = cpu_rq(other_cpu); if (rq_idle(rq)) continue; - if (!rq->online) + if (unlikely(!rq->online)) continue; p = rq->curr; if (!smt_should_schedule(p, this_rq)) { @@ -3552,7 +3406,7 @@ static void wake_smt_siblings(struct rq *this_rq) struct rq *rq; rq = cpu_rq(other_cpu); - if (!rq->online) + if (unlikely(!rq->online)) continue; if (rq_idle(rq)) { struct task_struct *p = rq->curr; @@ -3689,25 +3543,6 @@ static void __sched notrace __schedule(bool preempt) prev->deadline = rq->rq_deadline; check_deadline(prev); prev->last_ran = rq->clock_task; - - /* Task changed affinity off this CPU */ - if (likely(!needs_other_cpu(prev, cpu))) { - if (!deactivate) { - if (!queued_notrunning()) { - /* - * We now know prev is the only thing that is - * awaiting CPU so we can bypass rechecking for - * the earliest deadline task and just run it - * again. - */ - set_rq_task(rq, prev); - check_siblings(rq); - grq_unlock_irq(); - goto rerun_prev_unlocked; - } else - swap_sticky(rq, cpu, prev); - } - } return_task(prev, rq, deactivate); } @@ -3733,12 +3568,6 @@ static void __sched notrace __schedule(bool preempt) */ if (prev != idle && !deactivate) resched_suitable_idle(prev); - /* - * Don't stick tasks when a real time task is going to run as - * they may literally get stuck. - */ - if (rt_task(next)) - unstick_task(rq, prev); set_rq_task(rq, next); if (next != idle) check_siblings(rq); @@ -3758,9 +3587,6 @@ static void __sched notrace __schedule(bool preempt) check_siblings(rq); grq_unlock_irq(); } - -rerun_prev_unlocked: - return; } static inline void sched_submit_work(struct task_struct *tsk) @@ -5681,7 +5507,6 @@ static void bind_zero(int src_cpu) p->zerobound = true; bound++; } - clear_sticky(p); } while_each_thread(t, p); if (bound) { @@ -5956,8 +5781,11 @@ static void set_rq_online(struct rq *rq) static void set_rq_offline(struct rq *rq) { if (rq->online) { - cpumask_clear_cpu(cpu_of(rq), rq->rd->online); + int cpu = cpu_of(rq); + + cpumask_clear_cpu(cpu, rq->rd->online); rq->online = false; + clear_cpuidle_map(cpu); } } @@ -7347,8 +7175,6 @@ LIST_HEAD(task_groups); /* Cacheline aligned slab cache for task_group */ static struct kmem_cache *task_group_cache __read_mostly; -/* task_group_lock serializes the addition/removal of task groups */ -static DEFINE_SPINLOCK(task_group_lock); #endif /* CONFIG_CGROUP_SCHED */ void __init sched_init(void) @@ -7371,8 +7197,9 @@ void __init sched_init(void) grq.iso_ticks = 0; grq.iso_refractory = false; grq.noc = 1; - grq.node = skiplist_init(); - grq.sl = new_skiplist(grq.node); + skiplist_init(&grq.node); + grq.sl = new_skiplist(&grq.node); + skiplist_node_init(&init_task.node); #ifdef CONFIG_SMP init_defrootdomain(); @@ -7396,7 +7223,6 @@ void __init sched_init(void) rq->iowait_pc = rq->idle_pc = 0; rq->dither = false; #ifdef CONFIG_SMP - rq->sticky_task = NULL; rq->last_niffy = 0; rq->sd = NULL; rq->rd = NULL; @@ -7835,17 +7661,6 @@ struct task_group *sched_create_group(struct task_group *parent) void sched_online_group(struct task_group *tg, struct task_group *parent) { - unsigned long flags; - - spin_lock_irqsave(&task_group_lock, flags); - list_add_rcu(&tg->list, &task_groups); - - WARN_ON(!parent); /* root should already exist */ - - tg->parent = parent; - INIT_LIST_HEAD(&tg->children); - list_add_rcu(&tg->siblings, &parent->children); - spin_unlock_irqrestore(&task_group_lock, flags); } /* rcu callback to free various structures associated with a task group */ @@ -7863,12 +7678,6 @@ void sched_destroy_group(struct task_group *tg) void sched_offline_group(struct task_group *tg) { - unsigned long flags; - - spin_lock_irqsave(&task_group_lock, flags); - list_del_rcu(&tg->list); - list_del_rcu(&tg->siblings); - spin_unlock_irqrestore(&task_group_lock, flags); } static inline struct task_group *css_tg(struct cgroup_subsys_state *css) @@ -7916,7 +7725,7 @@ static void cpu_cgroup_fork(struct task_struct *task) static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) { - return 0; + return 0; } static void cpu_cgroup_attach(struct cgroup_taskset *tset) -- cgit v1.2.3